diff --git a/README.md b/README.md index d1fd9d1868f6fcdf21c94dc035d0684a94eac6a2..f7600eef3d658e0847fd499a9b310578b2b7241d 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,8 @@ Click flags top-right to switch. ## Local development +### Browser application + ```bash git clone https://github.com/karlesmarin/tafagent cd tafagent @@ -125,6 +127,24 @@ python -m http.server 8000 # open http://localhost:8000 ``` +### CLI diagnostic (for the paper) + +The directory `cli/diagnose_model.py` is the command-line companion +described in the paper *Transformer Thermodynamics* (Marin 2026). +It characterises any causal language model from HuggingFace in +minutes on CPU and produces the raw `gamma_obs`, `RΒ²`, and +thermodynamic profile used in the manuscript. + +```bash +pip install torch transformers numpy +python cli/diagnose_model.py --model EleutherAI/pythia-2.8b --fast --cpu +``` + +### Reproducibility data + +The directory `data/` ships every measurement referenced in the +paper (343 JSON files, ~5.5 MB). See `data/README.md` for the layout. + ## Browser requirements - **Chrome / Edge / Firefox 113+** for WebGPU acceleration (recommended) @@ -134,7 +154,7 @@ python -m http.server 8000 ## How you can help -This tool is at v0.2. There's a long way to go. +This tool is at v0.3. There's a long way to go. - **πŸ› Report bugs**: https://github.com/karlesmarin/tafagent/issues - **🌐 Translate**: add a language to `js/i18n.js`, send a PR diff --git a/cli/diagnose_model.py b/cli/diagnose_model.py new file mode 100644 index 0000000000000000000000000000000000000000..a5d36f636e1a542f07bc70937bfaac76c120328e --- /dev/null +++ b/cli/diagnose_model.py @@ -0,0 +1,438 @@ +""" +diagnose_model.py β€” Transformer Thermodynamics Diagnostic Tool +============================================================== +Single-command characterization of any causal LM via power-law attention decay. + +Measures: + Ξ³ (gamma) β€” attention decay exponent A(d) ∝ d^{-Ξ³} + T_attn = 1/Ξ³ β€” attention temperature + Phase β€” A (deconfined / RoPE), B (confined / AbsPE), C (ALiBi), Hagedorn + Z, U, S, F β€” thermodynamic potentials (partition function, energy, entropy, free energy) + C_V, Ο‡ β€” heat capacity, susceptibility + D_90 β€” context depth capturing 90% of Z (KV compression estimate) + Ξ”H_90 β€” holographic quality loss at D_90 + KL_grammar β€” attention grammar anomaly (deviation from power-law prior) + ΞΈ_eff β€” effective RoPE base (PadΓ© diagnostic) + Ξ³_pred β€” theoretical prediction C/ln(ΞΈ) where C=ln(10000)=9.2103 + +Usage: + python diagnose_model.py --model EleutherAI/pythia-70m + python diagnose_model.py --model meta-llama/Meta-Llama-3-8B --local /path/to/weights --load_in_4bit + python diagnose_model.py --model Qwen/Qwen2.5-7B --theta 1000000 --N 1000 + python diagnose_model.py --model EleutherAI/pythia-70m --fast # quick mode, 3 distances + +Output: + Prints diagnostic table to stdout. + Saves JSON to ./diagnose_results/{model_short}.json +""" + +import sys +sys.stdout.reconfigure(encoding='utf-8') +import argparse +import json +import math +import random +import time +from pathlib import Path + +import numpy as np + +# ── Constants ────────────────────────────────────────────────────────────────── +C_THEORY = math.log(10000) # 9.2103 β€” Ξ³ Γ— ln(ΞΈ) = C for standard RoPE +DISTANCES_FULL = [10, 20, 30, 50, 100, 200, 500, 1000, 2000] +DISTANCES_FAST = [10, 50, 200, 1000] +N_PROMPTS = 30 # per distance (fast mode default) +N_PROMPTS_FULL = 80 +SEEDS = [42, 123, 7] + +THETA_KNOWN = { + "EleutherAI/pythia-14m": 10_000, + "EleutherAI/pythia-31m": 10_000, + "EleutherAI/pythia-70m": 10_000, + "EleutherAI/pythia-160m": 10_000, + "EleutherAI/pythia-410m": 10_000, + "EleutherAI/pythia-1b": 10_000, + "EleutherAI/pythia-1.4b": 10_000, + "EleutherAI/pythia-2.8b": 10_000, + "mistralai/Mistral-7B-v0.1": 10_000, + "tiiuae/falcon-7b": 10_000, + "microsoft/phi-2": 10_000, + "meta-llama/Llama-2-7b-hf": 10_000, + "google/gemma-2-9b-it": 10_000, + "EleutherAI/gpt-j-6B": 10_000, + "meta-llama/Meta-Llama-3-8B": 500_000, + "Qwen/Qwen2.5-7B": 1_000_000, + "mistralai/Mistral-Nemo-Instruct-2407": 1_000_000, + "codellama/CodeLlama-13b-Instruct-hf": 1_000_000, +} + +OUTPUT_DIR = Path("./diagnose_results") + +# ── Thermodynamic functions ──────────────────────────────────────────────────── + +def partition_Z(gamma: float, N: int) -> float: + if abs(gamma - 1.0) < 1e-5: + return math.log(N + 0.5) + return (N ** (1 - gamma) - 1) / (1 - gamma) + 1 + + +def mean_log_d(gamma: float, N: int) -> float: + Z = partition_Z(gamma, N) + if Z <= 0: + return 0.0 + if abs(gamma - 1.0) < 1e-5: + integral = math.log(N) ** 2 / 2 + else: + g1 = 1.0 - gamma + integral = N ** g1 * (math.log(N) / g1 - 1 / g1 ** 2) + 1 / g1 ** 2 + return integral / Z + + +def entropy_S(gamma: float, N: int) -> float: + return math.log(partition_Z(gamma, N)) + gamma * mean_log_d(gamma, N) + + +def free_energy_F(gamma: float, N: int) -> float: + return -math.log(max(partition_Z(gamma, N), 1e-30)) + + +def heat_capacity_Cv(gamma: float, N: int, delta: float = 1e-4) -> float: + if gamma <= delta or gamma >= 20: + return float("nan") + dU = (mean_log_d(gamma + delta, N) - mean_log_d(gamma - delta, N)) / (2 * delta) + return -gamma ** 2 * dU + + +def D_f_closed(gamma: float, f: float, N: int) -> int: + if abs(gamma - 1.0) < 0.01: + return int(N * math.exp(math.log(f) / math.log(N))) + return max(1, min(N, int(N * f ** (1 / (1 - gamma))))) + + +def delta_H(theta: float, Df: int, N: int) -> float: + sqrt2 = math.sqrt(2) + return math.log((theta + Df / sqrt2) / (theta + N / sqrt2)) + + +def theta_eff_pade(theta: float, T: float) -> float: + return theta + T / math.sqrt(2) + + +def phase_label(gamma: float) -> str: + if gamma < 0.95: + return "A β€” deconfined (RoPE/long)" + if gamma > 1.05: + return "B β€” confined (AbsPE/short)" + return "Hagedorn (crossover Ξ³β‰ˆ1)" + + +def kl_divergence(p: np.ndarray, q: np.ndarray) -> float: + p = p / p.sum() + q = q / q.sum() + eps = 1e-12 + mask = p > eps + return float(np.sum(p[mask] * np.log(p[mask] / (q[mask] + eps)))) + + +# ── Attention measurement ────────────────────────────────────────────────────── + +def set_seed(seed: int): + random.seed(seed) + np.random.seed(seed) + try: + import torch + torch.manual_seed(seed) + if torch.cuda.is_available(): + torch.cuda.manual_seed_all(seed) + except ImportError: + pass + + +def measure_attn_distance(model, tokenizer, distance: int, n_prompts: int, + seed: int, device: str, vocab_high: int) -> float: + import torch + set_seed(seed) + rng = random.Random(seed) + seq_len = distance + 50 + target_pos = seq_len - distance - 1 + last_pos = seq_len - 1 + vocab_low = 1000 + + attn_values = [] + model.eval() + with torch.no_grad(): + for _ in range(n_prompts): + tokens = [rng.randint(vocab_low, vocab_high) for _ in range(seq_len)] + input_ids = torch.tensor([tokens], dtype=torch.long).to(device) + try: + out = model(input_ids, output_attentions=True, return_dict=True) + except Exception: + continue + if out.attentions is None: + raise RuntimeError( + "output_attentions returned None. " + "Try loading with attn_implementation='eager'." + ) + vals = [] + for layer_attn in out.attentions: + w = layer_attn[0, :, last_pos, target_pos].float().cpu().numpy() + finite = w[np.isfinite(w)] + if len(finite): + vals.append(float(np.mean(finite))) + if vals: + attn_values.append(float(np.mean(vals))) + + return float(np.mean(attn_values)) if attn_values else float("nan") + + +def fit_power_law(distances: list, means: list) -> dict: + d = np.array(distances, dtype=float) + m = np.array(means, dtype=float) + mask = np.isfinite(m) & (m > 0) + if mask.sum() < 2: + return {"gamma": float("nan"), "log_A": 0.0, "R2": 0.0} + log_d = np.log(d[mask]) + log_m = np.log(m[mask]) + X = np.stack([np.ones(mask.sum()), -log_d], axis=1) + coeffs, *_ = np.linalg.lstsq(X, log_m, rcond=None) + log_A, gamma = float(coeffs[0]), float(coeffs[1]) + pred = log_A - gamma * log_d + ss_res = float(np.sum((log_m - pred) ** 2)) + ss_tot = float(np.sum((log_m - np.mean(log_m)) ** 2)) + R2 = 1.0 - ss_res / ss_tot if ss_tot > 0 else 0.0 + return {"gamma": gamma, "log_A": log_A, "R2": round(R2, 6)} + + +# ── Attention Grammar anomaly ────────────────────────────────────────────────── + +def grammar_kl(attn_by_d: dict, gamma: float, log_A: float) -> float: + dists = sorted(attn_by_d.keys()) + p_obs = np.array([attn_by_d[d] for d in dists], dtype=float) + p_obs = np.maximum(p_obs, 1e-30) + p_obs /= p_obs.sum() + A = math.exp(log_A) + p_prior = np.array([A * d ** (-gamma) for d in dists], dtype=float) + p_prior = np.maximum(p_prior, 1e-30) + p_prior /= p_prior.sum() + return kl_divergence(p_obs, p_prior) + + +# ── Main diagnostic ─────────────────────────────────────────────────────────── + +def run_diagnostic(args) -> dict: + import torch + from transformers import AutoTokenizer, AutoModelForCausalLM + + model_name = args.model + theta_nom = args.theta or THETA_KNOWN.get(model_name, 10_000) + + print(f"\n{'='*65}") + print(f"TRANSFORMER THERMODYNAMICS DIAGNOSTIC") + print(f"{'='*65}") + print(f" Model : {model_name}") + print(f" theta_nom : {theta_nom:,}") + print(f" N : {args.N}") + print(f" Mode : {'fast' if args.fast else 'full'}") + print() + + # ── Load model ────────────────────────────────────────────────────── + local_path = args.local or model_name + print(f"Loading model from: {local_path} ...") + t0 = time.time() + + load_kwargs = dict( + trust_remote_code=True, + attn_implementation="eager", + ) + + device = "cuda" if (not args.cpu and torch.cuda.is_available()) else "cpu" + + if args.load_in_4bit and device == "cuda": + try: + from transformers import BitsAndBytesConfig + load_kwargs["quantization_config"] = BitsAndBytesConfig( + load_in_4bit=True, + bnb_4bit_compute_dtype=torch.float16, + bnb_4bit_use_double_quant=True, + ) + load_kwargs["device_map"] = "auto" + except ImportError: + print(" [warn] bitsandbytes not available; loading in float32") + elif device == "cpu": + load_kwargs["dtype"] = torch.float32 + + tokenizer = AutoTokenizer.from_pretrained(local_path, trust_remote_code=True) + model = AutoModelForCausalLM.from_pretrained(local_path, **load_kwargs) + if device == "cpu": + model = model.to("cpu") + model.eval() + print(f" Loaded in {time.time()-t0:.1f}s device={device}") + + vocab_high = min(tokenizer.vocab_size - 1, 49_000) + distances = DISTANCES_FAST if args.fast else DISTANCES_FULL + n_prompts = N_PROMPTS if args.fast else N_PROMPTS_FULL + N = args.N + + # ── Measure attention by distance ──────────────────────────────────── + print(f"\nMeasuring attention decay at {len(distances)} distances Γ— {n_prompts} prompts ...") + attn_by_d = {} + for dist in distances: + if dist > N: + continue + t1 = time.time() + mean_val = measure_attn_distance( + model, tokenizer, dist, n_prompts, SEEDS[0], device, vocab_high + ) + attn_by_d[dist] = mean_val + print(f" d={dist:5d} attn={mean_val:.6f} ({time.time()-t1:.1f}s)") + + # ── Fit power law ──────────────────────────────────────────────────── + valid_d = [d for d, v in attn_by_d.items() if math.isfinite(v) and v > 0] + valid_v = [attn_by_d[d] for d in valid_d] + fit = fit_power_law(valid_d, valid_v) + gamma = fit["gamma"] + log_A = fit["log_A"] + R2 = fit["R2"] + + if not math.isfinite(gamma): + print("\n[ERROR] Power-law fit failed. Too few valid distances.") + return {} + + # ── Thermodynamics ─────────────────────────────────────────────────── + Z = partition_Z(gamma, N) + U = mean_log_d(gamma, N) + S = entropy_S(gamma, N) + F = free_energy_F(gamma, N) + Cv = heat_capacity_Cv(gamma, N) + chi = 1.0 / abs(gamma - 1.0) if abs(gamma - 1.0) > 1e-4 else 1e6 + xi = 1.0 / abs(math.log(gamma)) if abs(math.log(gamma)) > 1e-10 else 1e6 + T_attn = 1.0 / gamma + + D90 = D_f_closed(gamma, 0.90, N) + dH90 = delta_H(theta_nom, D90, N) + theta_eff = theta_eff_pade(theta_nom, float(N)) + + # Theoretical gamma prediction + gamma_pred = C_THEORY / math.log(theta_nom) if theta_nom > 1 else None + + # Attention grammar KL + kl_ag = grammar_kl(attn_by_d, gamma, log_A) + + # Phase + phase = phase_label(gamma) + + # ── Report ─────────────────────────────────────────────────────────── + print(f"\n{'='*65}") + print(f"RESULTS") + print(f"{'='*65}") + print(f" Ξ³ (gamma) = {gamma:.4f} [RΒ²={R2:.4f}]") + if gamma_pred is not None: + delta_g = gamma - gamma_pred + print(f" Ξ³_pred (C/lnΞΈ) = {gamma_pred:.4f} Δγ = {delta_g:+.4f}") + print(f" Phase : {phase}") + print(f" T_attn = 1/Ξ³ = {T_attn:.4f}") + print() + print(f" Thermodynamics (N={N}):") + print(f" Z (partition) = {Z:.4f}") + print(f" U = E[log d] = {U:.4f}") + print(f" S (entropy) = {S:.4f}") + print(f" F (free ener) = {F:.4f}") + cv_str = f"{Cv:.4f}" if math.isfinite(Cv) else "N/A" + print(f" C_V (heat cap)= {cv_str}") + chi_str = f"{chi:.2f}" if chi < 1e5 else "∞ (near Hagedorn)" + print(f" Ο‡ (suscept.) = {chi_str}") + xi_str = f"{xi:.2f}" if xi < 1e5 else "∞" + print(f" ΞΎ (corr. len) = {xi_str}") + print() + print(f" KV Compression (f=0.90):") + print(f" D_90 = {D90} tokens ({D90/N*100:.1f}% of N={N})") + print(f" dH_90 = {dH90:.4f} nats") + print() + print(f" RoPE Diagnostic:") + print(f" theta_nom = {theta_nom:,}") + print(f" theta_eff_Pade = {theta_eff:.1f}") + print() + print(f" Attention Grammar:") + print(f" KL(obs||prior) = {kl_ag:.4f} ", end="") + if kl_ag > 0.05: + print("[HIGH β€” non-power-law circuits present]") + elif kl_ag > 0.01: + print("[MODERATE β€” some circuit deviation]") + else: + print("[LOW β€” pure positional attention]") + + print(f"\n Ξ³ interpretation:") + if gamma < 0.7: + print(f" Very long-range attention (large ΞΈ, LLaMA-3/Qwen2.5 class)") + elif gamma < 0.95: + print(f" Long-range attention (standard RoPE, Phase A)") + elif gamma < 1.05: + print(f" Hagedorn crossover β€” attention at phase boundary") + elif gamma < 1.3: + print(f" Short-range attention (AbsPE or short context training)") + else: + print(f" Highly local attention (possible SWA or very short context)") + + # ── Save ───────────────────────────────────────────────────────────── + OUTPUT_DIR.mkdir(parents=True, exist_ok=True) + short = model_name.replace("/", "--") + result = { + "model": model_name, + "theta_nom": theta_nom, + "N": N, + "fast_mode": args.fast, + "fit_power_law": fit, + "gamma": gamma, + "gamma_pred": gamma_pred, + "delta_gamma": (gamma - gamma_pred) if gamma_pred else None, + "phase": phase, + "T_attn": T_attn, + "Z": Z, "U": U, "S": S, "F": F, "Cv": Cv, + "chi": chi, "xi": xi, + "D90": D90, + "D90_frac": D90 / N, + "delta_H_90": dH90, + "theta_eff_pade": theta_eff, + "kl_grammar": kl_ag, + "attn_by_distance": {str(d): v for d, v in attn_by_d.items()}, + } + + out_path = OUTPUT_DIR / f"{short}.json" + out_path.write_text(json.dumps(result, indent=2, default=float), encoding="utf-8") + print(f"\n Saved: {out_path}") + print(f"{'='*65}\n") + + return result + + +def main(): + parser = argparse.ArgumentParser( + description="Transformer Thermodynamics Diagnostic β€” characterize any causal LM" + ) + parser.add_argument("--model", required=True, + help="HuggingFace model ID (e.g. EleutherAI/pythia-70m)") + parser.add_argument("--local", default=None, + help="Local path to model weights (if not downloading)") + parser.add_argument("--theta", type=int, default=None, + help="RoPE ΞΈ (auto-detected for known models)") + parser.add_argument("--N", type=int, default=2000, + help="Context length N for thermodynamic calculations (default 2000)") + parser.add_argument("--fast", action="store_true", + help="Fast mode: fewer distances and prompts (~5 min on CPU)") + parser.add_argument("--load_in_4bit", action="store_true", + help="Load model in 4-bit quantization (requires bitsandbytes)") + parser.add_argument("--cpu", action="store_true", + help="Force CPU even if CUDA available") + args = parser.parse_args() + + try: + run_diagnostic(args) + except KeyboardInterrupt: + print("\n[interrupted]") + except Exception as e: + print(f"\n[ERROR] {e}") + raise + + +if __name__ == "__main__": + main() diff --git a/data/README.md b/data/README.md new file mode 100644 index 0000000000000000000000000000000000000000..4c48a265ce9af295c6026ef5ea0cc3f4bdb37bdc --- /dev/null +++ b/data/README.md @@ -0,0 +1,61 @@ +# `data/` β€” Reproducibility artefacts for the TAF paper + +This directory ships the raw experimental outputs referenced in +*Transformer Thermodynamics: A Closed-Form Theory of Attention Decay, +Phase Transitions, and Context-Length Limits in RoPE Language Models* +(Marin 2026). + +Every claim in the paper that depends on a measurement is backed by +a JSON file under one of the subdirectories below. File contents are +deliberately verbose so a reader can verify a number without re-running +the experiment. + +## Layout + +| Subdirectory | Contents | +|---|---| +| `e4_gamma/` | $\gamma_\mathrm{obs}$ measurements per model (text + random corpus). 23 model entries. | +| `e1_h3/` | H3 residual transplant recoveries (paper Β§sec:structure). | +| `exp_b1/` | NIAH zero-shot context extension (paper Β§sec:ntk_scaling). | +| `exp_b2/` | KV-cache compression sweeps for `D_f` validity (paper Β§sec:kvcache). | +| `exp_b3/` | Dead-band vs alive-band ablation (paper Β§sec:gamma_dial). | +| `exp_kv_decay/` | Soft-decay vs hard-truncation regime panel (paper Β§sec:kv_horizon_decay). | +| `exp_wqk_spectral/` | DFT spectral analysis of $W_Q, W_K$ rows. | +| `exp_gamma_field/` | Per-layer per-head $\gamma$-field measurements. | +| `e7_e9_hagedorn/` | Phase-A/B Hagedorn boundary cross-models. | +| `e7_passkey/` | Passkey retrieval at variable distance. | +| `dict1_primitives/` | Primitive subspace clustering. | +| `dft_weights/` | Weight-space DFT signatures. | +| `attention_grammar/` | Attention grammar / KL-anomaly classifier. | +| `cloud/` | Long-running runs from cloud GPUs. | +| `master_gamma_results.json` | Curated 23-model summary (top-level). | +| `*.png` | Figures used in the paper. | + +## File format + +Each measurement file is a JSON object with at minimum: + +```json +{ + "model": "", + "corpus": "mongo" | "random", + "theta": , + "gamma_obs": , + "R2": , + "T_attn": +} +``` + +Additional fields (heat-capacity $C_V$, free energy $F$, decay +spectra, etc.) are present in experiment-specific files. + +## How to use this data + +To verify the PadΓ© prediction on a single model, point `cli/diagnose_model.py` at the same model identifier and compare its `gamma_obs` against the value here. To re-run an experiment from scratch, the originating Python scripts live in the parent paper repository (referenced from the manuscript appendices). + +## Excluded files + +For repository hygiene, this directory ships only `*.json`, `*.csv`, +and `*.png`. Run logs (`*.log`, `*.txt`) and intermediate artefacts +(`*.bak`, `*.stale`) are not committed; they live in the originating +experiment directory and are reproducible from the scripts. diff --git a/data/attention_grammar/attention_grammar.json b/data/attention_grammar/attention_grammar.json new file mode 100644 index 0000000000000000000000000000000000000000..de40eb9cc2cc032fa5932d91579964ab6f6740fd --- /dev/null +++ b/data/attention_grammar/attention_grammar.json @@ -0,0 +1,790 @@ +[ + { + "stem": "EleutherAI--pythia-1.4b_mongo", + "model": "EleutherAI/pythia-1.4b", + "corpus": "mongo", + "gamma": 0.7050725013322717, + "R2": 0.841258, + "global_kl_obs_vs_prior": 0.015196687632097014, + "global_jsd_obs_vs_prior": 0.004001526720341644, + "global_residuals_by_d": { + "10": 0.022810997760228713, + "20": 0.03228475244280296, + "30": -0.011224334795081836, + "50": -0.012844069302633723, + "100": -0.0070921871280450535, + "200": -0.010204827227022706, + "500": -0.013832530653058377, + "1000": -0.0062626191675716055, + "2000": 0.006364818070381502 + }, + "global_most_anomalous_d": [ + 20, + 10, + 500 + ], + "global_p_obs_by_d": { + "10": 0.37490817823881556, + "20": 0.24826526889618455, + "30": 0.15105285221844747, + "50": 0.10035373893700048, + "100": 0.06234466849163211, + "200": 0.03238854974138169, + "500": 0.008491135377199428, + "1000": 0.0074309779006989105, + "2000": 0.014764630198639783 + }, + "global_p_prior_by_d": { + "10": 0.35209718047858685, + "20": 0.2159805164533816, + "30": 0.1622771870135293, + "50": 0.1131978082396342, + "100": 0.06943685561967716, + "200": 0.042593376968404394, + "500": 0.022323666030257806, + "1000": 0.013693597068270516, + "2000": 0.00839981212825828 + }, + "global_excess_at_short": 0.031027346105316114, + "global_excess_at_long": -0.03102734610531624, + "layerwise": null + }, + { + "stem": "EleutherAI--pythia-14m_random", + "model": "EleutherAI/pythia-14m", + "corpus": "random", + "gamma": 1.003714187534367, + "R2": 0.977698, + "global_kl_obs_vs_prior": 0.038235433766210955, + "global_jsd_obs_vs_prior": 0.00927543307259299, + "global_residuals_by_d": { + "10": -0.109621215771189, + "20": -0.0012012943115310704, + "30": 0.016957189732496353, + "50": 0.05177433582179028, + "100": 0.020924402426756104, + "200": 0.014046340300907342, + "500": 0.006255652985175648, + "1000": -0.00042677764722707113, + "2000": 0.0012913664628212713 + }, + "global_most_anomalous_d": [ + 10, + 50, + 100 + ], + "global_p_obs_by_d": { + "10": 0.34234841552607276, + "20": 0.2242024765259773, + "30": 0.1670002392109769, + "50": 0.14162952124045466, + "100": 0.06573647877033136, + "200": 0.03639476879577857, + "500": 0.015164652948934263, + "1000": 0.004016269069116729, + "2000": 0.0035071779123574155 + }, + "global_p_prior_by_d": { + "10": 0.45196963129726175, + "20": 0.22540377083750837, + "30": 0.15004304947848054, + "50": 0.08985518541866439, + "100": 0.04481207634357525, + "200": 0.022348428494871227, + "500": 0.008908999963758615, + "1000": 0.0044430467163438, + "2000": 0.0022158114495361442 + }, + "global_excess_at_short": -0.042090984528433434, + "global_excess_at_long": 0.04209098452843329, + "layerwise": null + }, + { + "stem": "EleutherAI--pythia-160m_random", + "model": "EleutherAI/pythia-160m", + "corpus": "random", + "gamma": 1.0171452847779678, + "R2": 0.981723, + "global_kl_obs_vs_prior": 0.03671330395029776, + "global_jsd_obs_vs_prior": 0.009019862086648857, + "global_residuals_by_d": { + "10": -0.10364735955340265, + "20": 0.09603644351459367, + "30": -0.025229439149369162, + "50": 0.013286757041981095, + "100": 0.007907860305319384, + "200": 0.008381597655555666, + "500": 0.0028788036073672807, + "1000": 0.0007940899259680449, + "2000": -0.0004087533480132261 + }, + "global_most_anomalous_d": [ + 10, + 20, + 30 + ], + "global_p_obs_by_d": { + "10": 0.3525782481728875, + "20": 0.3214543529278542, + "30": 0.1240080810067092, + "50": 0.10204845762089101, + "100": 0.05176440110797339, + "200": 0.03005081037489671, + "500": 0.011411382834019048, + "1000": 0.005009978158898596, + "2000": 0.001674287795870391 + }, + "global_p_prior_by_d": { + "10": 0.45622560772629017, + "20": 0.2254179094132605, + "30": 0.14923752015607836, + "50": 0.08876170057890992, + "100": 0.043856540802654005, + "200": 0.021669212719341045, + "500": 0.008532579226651767, + "1000": 0.004215888232930551, + "2000": 0.002083041143883617 + }, + "global_excess_at_short": -0.019553598146197046, + "global_excess_at_long": 0.01955359814619715, + "layerwise": null + }, + { + "stem": "EleutherAI--pythia-1b_mongo", + "model": "EleutherAI/pythia-1b", + "corpus": "mongo", + "gamma": 0.9311078627189842, + "R2": 0.983104, + "global_kl_obs_vs_prior": 0.00216969110760508, + "global_jsd_obs_vs_prior": 0.0005501731235737363, + "global_residuals_by_d": { + "10": -0.01675299123311219, + "20": 0.01444989049216508, + "30": -0.005395896338322825, + "50": 0.008483313045061516, + "100": 0.0036134022999753865, + "200": -0.0012700273538523259, + "500": -0.0036562627430824918, + "1000": -0.00022986978657394287, + "2000": 0.0007584416177418246 + }, + "global_most_anomalous_d": [ + 10, + 20, + 50 + ], + "global_p_obs_by_d": { + "10": 0.41183879443150423, + "20": 0.23922718897198086, + "30": 0.1487005031520486, + "50": 0.10425284750745761, + "100": 0.053840256740840184, + "200": 0.0250717202716869, + "500": 0.007567009909828906, + "1000": 0.005656236740437108, + "2000": 0.003845442274215638 + }, + "global_p_prior_by_d": { + "10": 0.4285917856646164, + "20": 0.22477729847981578, + "30": 0.15409639949037143, + "50": 0.0957695344623961, + "100": 0.0502268544408648, + "200": 0.026341747625539227, + "500": 0.011223272652911398, + "1000": 0.005886106527011051, + "2000": 0.0030870006564738136 + }, + "global_excess_at_short": 0.0007843159657915794, + "global_excess_at_long": -0.0007843159657915494, + "layerwise": null + }, + { + "stem": "EleutherAI--pythia-2.8b_mongo", + "model": "EleutherAI/pythia-2.8b", + "corpus": "mongo", + "gamma": 0.6741618914822415, + "R2": 0.999287, + "global_kl_obs_vs_prior": 0.0007223693404512629, + "global_jsd_obs_vs_prior": 0.00018080660467757078, + "global_residuals_by_d": { + "10": -0.01723800016721183, + "20": 0.011123776406963604, + "30": 0.0015807634891239764, + "50": 0.0036786972847576704, + "100": 0.0008547629863665668 + }, + "global_most_anomalous_d": [ + 10, + 20, + 50 + ], + "global_p_obs_by_d": { + "10": 0.3596717196288746, + "20": 0.24733165868564272, + "30": 0.1812941258327714, + "50": 0.13103440015032247, + "100": 0.08066809570238882 + }, + "global_p_prior_by_d": { + "10": 0.37690971979608645, + "20": 0.23620788227867912, + "30": 0.17971336234364743, + "50": 0.1273557028655648, + "100": 0.07981333271602226 + }, + "global_excess_at_short": -0.0061142237602482274, + "global_excess_at_long": 0.006114223760248214, + "layerwise": null + }, + { + "stem": "EleutherAI--pythia-31m_mongo", + "model": "EleutherAI/pythia-31m", + "corpus": "mongo", + "gamma": 1.2350013988825523, + "R2": 0.973742, + "global_kl_obs_vs_prior": 0.023022979798356734, + "global_jsd_obs_vs_prior": 0.005471313991304833, + "global_residuals_by_d": { + "10": -0.08288755825759814, + "20": 0.010683334304791459, + "30": 0.009584346148507256, + "50": 0.042370637311097845, + "100": 0.008528697291978422, + "200": 0.005370974270641956, + "500": 0.005246411606574146, + "1000": 0.0012115456869942693, + "2000": -0.000108388362987054 + }, + "global_most_anomalous_d": [ + 10, + 50, + 20 + ], + "global_p_obs_by_d": { + "10": 0.4392815168778181, + "20": 0.23252273428055062, + "30": 0.1440358705668548, + "50": 0.11391611942129422, + "100": 0.03892422927208722, + "200": 0.018284275531661027, + "500": 0.009411083727282962, + "1000": 0.0029808736850458974, + "2000": 0.000643296637405384 + }, + "global_p_prior_by_d": { + "10": 0.5221690751354162, + "20": 0.22183939997575916, + "30": 0.13445152441834754, + "50": 0.07154548211019637, + "100": 0.030395531980108796, + "200": 0.012913301261019071, + "500": 0.004164672120708816, + "1000": 0.0017693279980516281, + "2000": 0.000751685000392438 + }, + "global_excess_at_short": -0.02024924049320158, + "global_excess_at_long": 0.020249240493201738, + "layerwise": null + }, + { + "stem": "EleutherAI--pythia-410m_mongo", + "model": "EleutherAI/pythia-410m", + "corpus": "mongo", + "gamma": 1.0218530106365162, + "R2": 0.981594, + "global_kl_obs_vs_prior": 0.019950329075816703, + "global_jsd_obs_vs_prior": 0.0049277310238796685, + "global_residuals_by_d": { + "10": -0.08643099400783588, + "20": 0.037217492153397425, + "30": -0.003010973769102726, + "50": 0.024826432875333385, + "100": 0.009989029001931292, + "200": 0.011247915678530299, + "500": 0.004870809845053985, + "1000": 0.0015369498921579995, + "2000": -0.0002466616694658803 + }, + "global_most_anomalous_d": [ + 10, + 20, + 50 + ], + "global_p_obs_by_d": { + "10": 0.3712812388310567, + "20": 0.26263316683164134, + "30": 0.1459404740627973, + "50": 0.11320519539681292, + "100": 0.0535141020755623, + "200": 0.03268329146357251, + "500": 0.013274981332990713, + "1000": 0.0056758648045145774, + "2000": 0.0017916852010515516 + }, + "global_p_prior_by_d": { + "10": 0.45771223283889256, + "20": 0.22541567467824392, + "30": 0.14895144783190004, + "50": 0.08837876252147954, + "100": 0.04352507307363101, + "200": 0.02143537578504221, + "500": 0.008404171487936728, + "1000": 0.004138914912356578, + "2000": 0.002038346870517432 + }, + "global_excess_at_short": -0.0273980427482078, + "global_excess_at_long": 0.027398042748207695, + "layerwise": null + }, + { + "stem": "EleutherAI--pythia-70m_mongo", + "model": "EleutherAI/pythia-70m", + "corpus": "mongo", + "gamma": 0.7476017873166874, + "R2": 0.984269, + "global_kl_obs_vs_prior": 0.005536320533261201, + "global_jsd_obs_vs_prior": 0.0013726883907973612, + "global_residuals_by_d": { + "10": -0.0311974747311054, + "20": 0.03653998343007209, + "30": -0.0068868247326356324, + "50": 0.008032257936625584, + "100": -0.0040606113434220395, + "200": 0.0014190901714025428, + "500": -0.0030040427298673386, + "1000": -0.002436798284709307, + "2000": 0.0015944202836393933 + }, + "global_most_anomalous_d": [ + 20, + 10, + 50 + ], + "global_p_obs_by_d": { + "10": 0.33568322810408546, + "20": 0.25505148758761964, + "30": 0.1544851696977996, + "50": 0.118179504403571, + "100": 0.061542295318900604, + "200": 0.040491708875828765, + "500": 0.016691677086524588, + "1000": 0.009293830579896667, + "2000": 0.008581098345773568 + }, + "global_p_prior_by_d": { + "10": 0.36688070283519086, + "20": 0.21851150415754755, + "30": 0.16137199443043523, + "50": 0.11014724646694542, + "100": 0.06560290666232264, + "200": 0.03907261870442622, + "500": 0.019695719816391927, + "1000": 0.011730628864605974, + "2000": 0.006986678062134175 + }, + "global_excess_at_short": 0.006487941902956637, + "global_excess_at_long": -0.006487941902956749, + "layerwise": null + }, + { + "stem": "google--gemma-2-9b-it_mongo", + "model": "google/gemma-2-9b-it", + "corpus": "mongo", + "gamma": 0.6276459084140061, + "R2": 0.977314, + "global_kl_obs_vs_prior": 0.00990710261366268, + "global_jsd_obs_vs_prior": 0.002518861928032747, + "global_residuals_by_d": { + "10": 0.04775177103579237, + "20": 0.008660386938166148, + "30": -0.002496755850920934, + "50": -0.018987961296649064, + "100": -0.017002580781742056, + "200": -0.013143143872362165, + "500": -0.004174428437412087, + "1000": 0.0020818104433162343, + "2000": -0.002689098178188208 + }, + "global_most_anomalous_d": [ + 10, + 50, + 100 + ], + "global_p_obs_by_d": { + "10": 0.3725548286554262, + "20": 0.21888320997930266, + "30": 0.16049172263627337, + "50": 0.09929289109441769, + "100": 0.059552531979791705, + "200": 0.036405749050342044, + "500": 0.02370401599192719, + "1000": 0.020125622385488293, + "2000": 0.008989428227030857 + }, + "global_p_prior_by_d": { + "10": 0.32480305761963385, + "20": 0.21022282304113651, + "30": 0.1629884784871943, + "50": 0.11828085239106675, + "100": 0.07655511276153376, + "200": 0.04954889292270421, + "500": 0.027878444429339278, + "1000": 0.01804381194217206, + "2000": 0.011678526405219065 + }, + "global_excess_at_short": 0.03492744082638852, + "global_excess_at_long": -0.03492744082638828, + "layerwise": null + }, + { + "stem": "google--gemma-2-9b-it_random", + "model": "google/gemma-2-9b-it", + "corpus": "random", + "gamma": 1.1347958464287666, + "R2": 0.976472, + "global_kl_obs_vs_prior": 0.07449950182862176, + "global_jsd_obs_vs_prior": 0.017657713701983022, + "global_residuals_by_d": { + "10": -0.151231598586703, + "20": -0.0027642922764635203, + "30": 0.027636455378714536, + "50": 0.059516548386964865, + "100": 0.039447447904125695, + "200": 0.018929065402736887, + "500": 0.005351562239854117, + "1000": 0.002959550440207742, + "2000": 0.00015526111056246442 + }, + "global_most_anomalous_d": [ + 10, + 50, + 100 + ], + "global_p_obs_by_d": { + "10": 0.34133602858891476, + "20": 0.2215506874842362, + "30": 0.16922582472899156, + "50": 0.1388173555454561, + "100": 0.07556098434329964, + "200": 0.03537514660764801, + "500": 0.011165652389049046, + "1000": 0.005607283344665459, + "2000": 0.0013610369677391604 + }, + "global_p_prior_by_d": { + "10": 0.49256762717561775, + "20": 0.22431497976069972, + "30": 0.14158936935027702, + "50": 0.07930080715849123, + "100": 0.03611353643917394, + "200": 0.016446081204911124, + "500": 0.005814090149194929, + "1000": 0.002647732904457717, + "2000": 0.001205775857176696 + }, + "global_excess_at_short": -0.0668428870974871, + "global_excess_at_long": 0.0668428870974869, + "layerwise": null + }, + { + "stem": "meta-llama--Llama-2-7b-hf_mongo", + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "mongo", + "gamma": 0.2870574377368437, + "R2": 0.814928, + "global_kl_obs_vs_prior": 0.03791116862626136, + "global_jsd_obs_vs_prior": 0.009505827750478257, + "global_residuals_by_d": { + "10": 0.09203177246740807, + "20": 0.015301495050760533, + "30": 0.0022298864801771068, + "50": -0.017585828555110566, + "100": -0.026931918267808955, + "200": -0.03634242461713364, + "500": -0.02216221333359098, + "1000": -0.007399582989034897, + "2000": 0.0008588137643333466 + }, + "global_most_anomalous_d": [ + 10, + 200, + 100 + ], + "global_p_obs_by_d": { + "10": 0.2949091819079361, + "20": 0.18157413470176506, + "30": 0.1502335569001011, + "50": 0.11023132807977587, + "100": 0.07782344155749717, + "200": 0.04951213279158751, + "500": 0.04383592944661877, + "1000": 0.04669064540254072, + "2000": 0.04518964921217767 + }, + "global_p_prior_by_d": { + "10": 0.202877409440528, + "20": 0.16627263965100453, + "30": 0.148003670419924, + "50": 0.12781715663488644, + "100": 0.10475535982530612, + "200": 0.08585455740872115, + "500": 0.06599814278020975, + "1000": 0.054090228391575616, + "2000": 0.04433083544784432 + }, + "global_excess_at_short": 0.09197732544323514, + "global_excess_at_long": -0.09197732544323513, + "layerwise": null + }, + { + "stem": "meta-llama--Llama-2-7b-hf_random", + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "random", + "gamma": 0.8266242679750889, + "R2": 0.993628, + "global_kl_obs_vs_prior": 0.018687030914933386, + "global_jsd_obs_vs_prior": 0.0045711108436779415, + "global_residuals_by_d": { + "10": -0.0673889176608512, + "20": -0.018845838498142836, + "30": 0.014738802602548395, + "50": 0.026738085304395573, + "100": 0.02700958949274316, + "200": 0.008969393861870123, + "500": 0.0063405366504610036, + "1000": 0.0016120084431194715, + "2000": 0.0008263398038562018 + }, + "global_most_anomalous_d": [ + 10, + 100, + 50 + ], + "global_p_obs_by_d": { + "10": 0.3265034073448185, + "20": 0.2032491806389243, + "30": 0.17358525163569818, + "50": 0.1308719413515151, + "100": 0.08572515535523795, + "200": 0.04207599151818851, + "500": 0.021863251603493377, + "1000": 0.010364445227723843, + "2000": 0.005761375324400295 + }, + "global_p_prior_by_d": { + "10": 0.3938923250056697, + "20": 0.22209501913706714, + "30": 0.1588464490331498, + "50": 0.10413385604711951, + "100": 0.05871556586249479, + "200": 0.03310659765631839, + "500": 0.015522714953032373, + "1000": 0.008752436784604372, + "2000": 0.004935035520544093 + }, + "global_excess_at_short": -0.04475786825205007, + "global_excess_at_long": 0.04475786825204996, + "layerwise": null + }, + { + "stem": "meta-llama--Meta-Llama-3-8B_mongo", + "model": "meta-llama/Meta-Llama-3-8B", + "corpus": "mongo", + "gamma": 1.0454762537473639, + "R2": 0.997461, + "global_kl_obs_vs_prior": 0.010104511537707845, + "global_jsd_obs_vs_prior": 0.0025338449662697608, + "global_residuals_by_d": { + "10": -0.06907195192290344, + "20": 0.021641853427447316, + "30": 0.025108972343980324, + "50": 0.0122513818444661, + "100": 0.004758070613705076, + "200": 0.0046074966616670154, + "500": -0.0002836402419261778, + "1000": 0.0005685537011081362, + "2000": 0.0004192635724557211 + }, + "global_most_anomalous_d": [ + 10, + 30, + 20 + ], + "global_p_obs_by_d": { + "10": 0.3960596794910087, + "20": 0.24699112714218127, + "30": 0.17259705603195316, + "50": 0.09871219848616582, + "100": 0.046647034767277044, + "200": 0.024902069232393336, + "500": 0.007502873949730214, + "1000": 0.004341002615247398, + "2000": 0.002246958284043105 + }, + "global_p_prior_by_d": { + "10": 0.4651316314139121, + "20": 0.22534927371473396, + "30": 0.14748808368797284, + "50": 0.08646081664169972, + "100": 0.04188896415357197, + "200": 0.02029457257072632, + "500": 0.007786514191656392, + "1000": 0.003772448914139262, + "2000": 0.0018276947115873838 + }, + "global_excess_at_short": -0.010069744307009701, + "global_excess_at_long": 0.01006974430700977, + "layerwise": null + }, + { + "stem": "mistralai--Mistral-7B-v0.1_mongo", + "model": "mistralai/Mistral-7B-v0.1", + "corpus": "mongo", + "gamma": 1.060750419523944, + "R2": 0.99869, + "global_kl_obs_vs_prior": 0.016298823278231955, + "global_jsd_obs_vs_prior": 0.00404723341503832, + "global_residuals_by_d": { + "10": -0.08138091364930577, + "20": 0.007076503197307182, + "30": 0.03039283128138065, + "50": 0.025632025650860563, + "100": 0.011293570532788226, + "200": 0.0031204868467434276, + "500": 0.0023678153968065448, + "1000": 0.0012137964564334605, + "2000": 0.00028388428698590496 + }, + "global_most_anomalous_d": [ + 10, + 30, + 50 + ], + "global_p_obs_by_d": { + "10": 0.38851182634611714, + "20": 0.23233492698957284, + "30": 0.1769112301011724, + "50": 0.11085682938197941, + "100": 0.05214886222481146, + "200": 0.02270580457309954, + "500": 0.009777770732282054, + "1000": 0.004766000432295189, + "2000": 0.001986749218670158 + }, + "global_p_prior_by_d": { + "10": 0.4698927399954229, + "20": 0.22525842379226566, + "30": 0.14651839881979176, + "50": 0.08522480373111885, + "100": 0.04085529169202323, + "200": 0.019585317726356112, + "500": 0.0074099553354755095, + "1000": 0.003552203975861728, + "2000": 0.0017028649316842529 + }, + "global_excess_at_short": -0.01827955351975738, + "global_excess_at_long": 0.018279553519757564, + "layerwise": null + }, + { + "stem": "mistralai--Mistral-7B-v0.1_random", + "model": "mistralai/Mistral-7B-v0.1", + "corpus": "random", + "gamma": 0.8296009929924347, + "R2": 0.996923, + "global_kl_obs_vs_prior": 0.011311151481069133, + "global_jsd_obs_vs_prior": 0.0027910372924293382, + "global_residuals_by_d": { + "10": -0.054361247230578225, + "20": -0.015024519249714402, + "30": 0.020394287518846393, + "50": 0.021609080199683883, + "100": 0.018730410355522142, + "200": 0.0021788109911298204, + "500": 0.004196456328857864, + "1000": 0.001453639002462927, + "2000": 0.0008230820837896334 + }, + "global_most_anomalous_d": [ + 10, + 50, + 30 + ], + "global_p_obs_by_d": { + "10": 0.3405361631650987, + "20": 0.20717826808071257, + "30": 0.17912611601544295, + "50": 0.1255096851311888, + "100": 0.0771937059187048, + "200": 0.03507522127056269, + "500": 0.019578607782364945, + "1000": 0.010108942476555976, + "2000": 0.005693290159368489 + }, + "global_p_prior_by_d": { + "10": 0.39489741039567694, + "20": 0.22220278733042698, + "30": 0.15873182849659656, + "50": 0.10390060493150491, + "100": 0.05846329556318265, + "200": 0.03289641027943287, + "500": 0.015382151453507081, + "1000": 0.00865530347409305, + "2000": 0.004870208075578856 + }, + "global_excess_at_short": -0.02738239876176235, + "global_excess_at_long": 0.027382398761762388, + "layerwise": null + }, + { + "stem": "Qwen--Qwen2.5-7B_mongo", + "model": "Qwen/Qwen2.5-7B", + "corpus": "mongo", + "gamma": 0.9966953735480816, + "R2": 0.993942, + "global_kl_obs_vs_prior": 0.0018136915141001462, + "global_jsd_obs_vs_prior": 0.0004515428142948941, + "global_residuals_by_d": { + "10": -0.024701063990131777, + "20": 0.006251851470610387, + "30": 0.008709334783999684, + "50": 0.007144885615584823, + "100": 0.00400098973421488, + "200": -0.002577353492273147, + "500": -9.69236465223431e-05, + "1000": 0.0013701858352996822, + "2000": -0.00010190631078206775 + }, + "global_most_anomalous_d": [ + 10, + 30, + 50 + ], + "global_p_obs_by_d": { + "10": 0.42503587951954835, + "20": 0.23163599563433493, + "30": 0.15916689558952096, + "50": 0.09757194214618171, + "100": 0.04931820247272019, + "200": 0.02013321399912888, + "500": 0.009014852064262025, + "1000": 0.0059365213343595775, + "2000": 0.0021864972399435723 + }, + "global_p_prior_by_d": { + "10": 0.44973694350968013, + "20": 0.22538414416372454, + "30": 0.15045756080552128, + "50": 0.09042705653059689, + "100": 0.04531721273850531, + "200": 0.022710567491402028, + "500": 0.009111775710784368, + "1000": 0.004566335499059895, + "2000": 0.00228840355072564 + }, + "global_excess_at_short": -0.002594992119936884, + "global_excess_at_long": 0.0025949921199370043, + "layerwise": null + } +] \ No newline at end of file diff --git a/data/bloom_extrapolation/bloom_extrapolation.json b/data/bloom_extrapolation/bloom_extrapolation.json new file mode 100644 index 0000000000000000000000000000000000000000..cc9f87b69bb9905febc43671ca042976e8c225d4 --- /dev/null +++ b/data/bloom_extrapolation/bloom_extrapolation.json @@ -0,0 +1,35 @@ +{ + "random": { + "model": "bloom-7b1", + "training_window": 2048, + "distances": [ + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000, + 3000 + ], + "means": [ + 0.011754, + 0.006069, + 0.002389, + 0.00128, + 0.000567, + 0.000224, + 8.2e-05, + 2e-05, + 6e-06 + ], + "gamma_short": 1.1505681472053984, + "r2_short": 0.9847279057720187, + "gamma_local_extrapolation": 2.969362295916118, + "delta_gamma": 1.8187941487107198, + "n_prompts": 15, + "seed": 42, + "note": "d=5000,8000 failed OOM (seq^2 attention matrix). d=3000 is only extrapolation point." + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/combined_training_dynamics.json b/data/checkpoint_eval/combined_training_dynamics.json new file mode 100644 index 0000000000000000000000000000000000000000..f8caf41f73504f2a97f4b4243e1b366909a78af7 --- /dev/null +++ b/data/checkpoint_eval/combined_training_dynamics.json @@ -0,0 +1,163 @@ +[ + { + "step": 512, + "gamma": 0.855, + "r2": 0.9161, + "lambada_ppl": 116756.3, + "lambada_acc": null + }, + { + "step": 1000, + "gamma": 0.8565, + "r2": 0.9752, + "lambada_ppl": 4465.3, + "lambada_acc": 0.05 + }, + { + "step": 2000, + "gamma": 0.9048, + "r2": 0.9885, + "lambada_ppl": null, + "lambada_acc": null + }, + { + "step": 3000, + "gamma": null, + "r2": null, + "lambada_ppl": 411.7, + "lambada_acc": 0.121 + }, + { + "step": 4000, + "gamma": 0.9832, + "r2": 0.9887, + "lambada_ppl": null, + "lambada_acc": null + }, + { + "step": 8000, + "gamma": 1.0636, + "r2": 0.9897, + "lambada_ppl": null, + "lambada_acc": null + }, + { + "step": 13000, + "gamma": null, + "r2": null, + "lambada_ppl": 136.2, + "lambada_acc": 0.21 + }, + { + "step": 16000, + "gamma": 1.002, + "r2": 0.9918, + "lambada_ppl": null, + "lambada_acc": null + }, + { + "step": 23000, + "gamma": null, + "r2": null, + "lambada_ppl": 121.3, + "lambada_acc": 0.223 + }, + { + "step": 32000, + "gamma": 0.8865, + "r2": 0.9912, + "lambada_ppl": null, + "lambada_acc": null + }, + { + "step": 33000, + "gamma": null, + "r2": null, + "lambada_ppl": 118.1, + "lambada_acc": 0.223 + }, + { + "step": 43000, + "gamma": null, + "r2": null, + "lambada_ppl": 112.4, + "lambada_acc": 0.228 + }, + { + "step": 53000, + "gamma": null, + "r2": null, + "lambada_ppl": 94.3, + "lambada_acc": 0.253 + }, + { + "step": 63000, + "gamma": null, + "r2": null, + "lambada_ppl": 101.7, + "lambada_acc": 0.233 + }, + { + "step": 64000, + "gamma": 0.7948, + "r2": 0.9908, + "lambada_ppl": null, + "lambada_acc": null + }, + { + "step": 73000, + "gamma": null, + "r2": null, + "lambada_ppl": 117.1, + "lambada_acc": 0.226 + }, + { + "step": 83000, + "gamma": null, + "r2": null, + "lambada_ppl": 124.3, + "lambada_acc": 0.226 + }, + { + "step": 93000, + "gamma": null, + "r2": null, + "lambada_ppl": 140.5, + "lambada_acc": 0.214 + }, + { + "step": 103000, + "gamma": null, + "r2": null, + "lambada_ppl": 125.6, + "lambada_acc": 0.226 + }, + { + "step": 113000, + "gamma": null, + "r2": null, + "lambada_ppl": 120.5, + "lambada_acc": 0.216 + }, + { + "step": 123000, + "gamma": null, + "r2": null, + "lambada_ppl": 133.1, + "lambada_acc": 0.204 + }, + { + "step": 133000, + "gamma": null, + "r2": null, + "lambada_ppl": 148.5, + "lambada_acc": 0.193 + }, + { + "step": 143000, + "gamma": 0.7805, + "r2": 0.9882, + "lambada_ppl": 142.4, + "lambada_acc": 0.185 + } +] \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_benchmarks_pythia70m.json b/data/checkpoint_eval/eleutherai_benchmarks_pythia70m.json new file mode 100644 index 0000000000000000000000000000000000000000..52d69d20a1fcb20110175483d3e10cd676344e5b --- /dev/null +++ b/data/checkpoint_eval/eleutherai_benchmarks_pythia70m.json @@ -0,0 +1,272 @@ +[ + { + "step": 0, + "lambada_ppl": 3684074.4512471026, + "lambada_acc": 0.0, + "piqa": 0.5255712731229597, + "sciq": 0.194, + "arc_easy": 0.2718855218855219, + "arc_challenge": 0.24658703071672355, + "winogrande": 0.4988161010260458 + }, + { + "step": 1, + "lambada_ppl": 3684074.4512471026, + "lambada_acc": 0.0, + "piqa": 0.5255712731229597, + "sciq": 0.194, + "arc_easy": 0.2718855218855219, + "arc_challenge": 0.24658703071672355, + "winogrande": 0.4988161010260458 + }, + { + "step": 2, + "lambada_ppl": 3683856.625077963, + "lambada_acc": 0.0, + "piqa": 0.5228509249183896, + "sciq": 0.194, + "arc_easy": 0.27104377104377103, + "arc_challenge": 0.24658703071672355, + "winogrande": 0.4980268350434096 + }, + { + "step": 4, + "lambada_ppl": 3681578.6002349453, + "lambada_acc": 0.0, + "piqa": 0.5261153427638737, + "sciq": 0.193, + "arc_easy": 0.27230639730639733, + "arc_challenge": 0.24573378839590443, + "winogrande": 0.49329123914759276 + }, + { + "step": 8, + "lambada_ppl": 3646061.8456566, + "lambada_acc": 0.0, + "piqa": 0.5239390642002176, + "sciq": 0.201, + "arc_easy": 0.26641414141414144, + "arc_challenge": 0.24488054607508533, + "winogrande": 0.489344909234412 + }, + { + "step": 16, + "lambada_ppl": 3526086.6936456356, + "lambada_acc": 0.0, + "piqa": 0.5266594124047879, + "sciq": 0.205, + "arc_easy": 0.26262626262626265, + "arc_challenge": 0.24829351535836178, + "winogrande": 0.48224151539068666 + }, + { + "step": 32, + "lambada_ppl": 3288862.4386760374, + "lambada_acc": 0.0, + "piqa": 0.5272034820457019, + "sciq": 0.223, + "arc_easy": 0.2668350168350168, + "arc_challenge": 0.2440273037542662, + "winogrande": 0.4940805051302289 + }, + { + "step": 64, + "lambada_ppl": 2347965.083490206, + "lambada_acc": 0.0, + "piqa": 0.5402611534276387, + "sciq": 0.199, + "arc_easy": 0.2706228956228956, + "arc_challenge": 0.24488054607508533, + "winogrande": 0.4972375690607735 + }, + { + "step": 128, + "lambada_ppl": 1665636.981895382, + "lambada_acc": 0.0, + "piqa": 0.5261153427638737, + "sciq": 0.219, + "arc_easy": 0.2676767676767677, + "arc_challenge": 0.23890784982935154, + "winogrande": 0.49171270718232046 + }, + { + "step": 256, + "lambada_ppl": 705314.6370389248, + "lambada_acc": 0.0, + "piqa": 0.5179542981501633, + "sciq": 0.228, + "arc_easy": 0.27441077441077444, + "arc_challenge": 0.24146757679180889, + "winogrande": 0.4925019731649566 + }, + { + "step": 512, + "lambada_ppl": 116756.33428953367, + "lambada_acc": 0.0, + "piqa": 0.5386289445048966, + "sciq": 0.264, + "arc_easy": 0.2984006734006734, + "arc_challenge": 0.2175767918088737, + "winogrande": 0.5098658247829518 + }, + { + "step": 1000, + "lambada_ppl": 4465.3093044480365, + "lambada_acc": 0.050067921599068504, + "piqa": 0.5603917301414582, + "sciq": 0.452, + "arc_easy": 0.3085016835016835, + "arc_challenge": 0.21160409556313994, + "winogrande": 0.5114443567482242 + }, + { + "step": 3000, + "lambada_ppl": 411.658325603736, + "lambada_acc": 0.12128856976518533, + "piqa": 0.5794341675734495, + "sciq": 0.592, + "arc_easy": 0.35058922558922556, + "arc_challenge": 0.21245733788395904, + "winogrande": 0.5082872928176796 + }, + { + "step": 13000, + "lambada_ppl": 136.24789967804702, + "lambada_acc": 0.21036289540073744, + "piqa": 0.5865070729053319, + "sciq": 0.65, + "arc_easy": 0.3952020202020202, + "arc_challenge": 0.22184300341296928, + "winogrande": 0.5027624309392266 + }, + { + "step": 23000, + "lambada_ppl": 121.29880202709046, + "lambada_acc": 0.2225887832330681, + "piqa": 0.6077257889009793, + "sciq": 0.653, + "arc_easy": 0.3952020202020202, + "arc_challenge": 0.20477815699658702, + "winogrande": 0.48539857932123126 + }, + { + "step": 33000, + "lambada_ppl": 118.09596009074914, + "lambada_acc": 0.2233650300795653, + "piqa": 0.5984766050054406, + "sciq": 0.664, + "arc_easy": 0.4010942760942761, + "arc_challenge": 0.21416382252559726, + "winogrande": 0.4972375690607735 + }, + { + "step": 43000, + "lambada_ppl": 112.36318862751271, + "lambada_acc": 0.22821657287017272, + "piqa": 0.5903155603917302, + "sciq": 0.663, + "arc_easy": 0.39941077441077444, + "arc_challenge": 0.20477815699658702, + "winogrande": 0.5027624309392266 + }, + { + "step": 53000, + "lambada_ppl": 94.31955728859376, + "lambada_acc": 0.25344459538133124, + "piqa": 0.5919477693144722, + "sciq": 0.664, + "arc_easy": 0.39225589225589225, + "arc_challenge": 0.21331058020477817, + "winogrande": 0.494869771112865 + }, + { + "step": 63000, + "lambada_ppl": 101.68439461161867, + "lambada_acc": 0.23287405394915583, + "piqa": 0.5930359085963003, + "sciq": 0.681, + "arc_easy": 0.4090909090909091, + "arc_challenge": 0.20733788395904437, + "winogrande": 0.4996053670086819 + }, + { + "step": 73000, + "lambada_ppl": 117.09850923121336, + "lambada_acc": 0.22627595575392975, + "piqa": 0.5984766050054406, + "sciq": 0.696, + "arc_easy": 0.4090909090909091, + "arc_challenge": 0.22098976109215018, + "winogrande": 0.5146014206787688 + }, + { + "step": 83000, + "lambada_ppl": 124.26962204175287, + "lambada_acc": 0.22627595575392975, + "piqa": 0.5973884657236126, + "sciq": 0.633, + "arc_easy": 0.37415824915824913, + "arc_challenge": 0.22440273037542663, + "winogrande": 0.5193370165745856 + }, + { + "step": 93000, + "lambada_ppl": 140.52328755411287, + "lambada_acc": 0.21405006792159906, + "piqa": 0.5979325353645266, + "sciq": 0.642, + "arc_easy": 0.359006734006734, + "arc_challenge": 0.22013651877133106, + "winogrande": 0.5082872928176796 + }, + { + "step": 103000, + "lambada_ppl": 125.56792285427366, + "lambada_acc": 0.22608189404230544, + "piqa": 0.6022850924918389, + "sciq": 0.628, + "arc_easy": 0.3707912457912458, + "arc_challenge": 0.2226962457337884, + "winogrande": 0.5098658247829518 + }, + { + "step": 113000, + "lambada_ppl": 120.54804436871059, + "lambada_acc": 0.21618474674946633, + "piqa": 0.5914036996735582, + "sciq": 0.617, + "arc_easy": 0.3720538720538721, + "arc_challenge": 0.21928327645051193, + "winogrande": 0.5185477505919495 + }, + { + "step": 123000, + "lambada_ppl": 133.11058088169239, + "lambada_acc": 0.20415292062875995, + "piqa": 0.5859630032644179, + "sciq": 0.614, + "arc_easy": 0.36574074074074076, + "arc_challenge": 0.23037542662116042, + "winogrande": 0.5027624309392266 + }, + { + "step": 133000, + "lambada_ppl": 148.4586759416483, + "lambada_acc": 0.19347952648942363, + "piqa": 0.5984766050054406, + "sciq": 0.617, + "arc_easy": 0.3707912457912458, + "arc_challenge": 0.22013651877133106, + "winogrande": 0.5232833464877664 + }, + { + "step": 143000, + "lambada_ppl": 142.42891015470678, + "lambada_acc": 0.18455268775470599, + "piqa": 0.5946681175190425, + "sciq": 0.601, + "arc_easy": 0.37373737373737376, + "arc_challenge": 0.22098976109215018, + "winogrande": 0.5280189423835833 + } +] \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step0.json b/data/checkpoint_eval/eleutherai_evals/step0.json new file mode 100644 index 0000000000000000000000000000000000000000..556868be8d280d8ef35e44503ea310c382410166 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step0.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 10.157118055555555, + "likelihood_difference_stderr": 1.3610391208223753, + "pct_stereotype": 0.5138888888888888, + "pct_stereotype_stderr": 0.05931618532716555 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.21794871794871795, + "acc_stderr": 0.020932445774463196, + "acc_norm": 0.24358974358974358, + "acc_norm_stderr": 0.02176373368417392 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2765957446808511, + "acc_stderr": 0.026684564340460997, + "acc_norm": 0.2624113475177305, + "acc_norm_stderr": 0.026244920349843007 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.18543046357615894, + "acc_stderr": 0.031732843842942865, + "acc_norm": 0.31125827814569534, + "acc_norm_stderr": 0.03780445850526733 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.844758064516129, + "likelihood_difference_stderr": 0.7849858741881288, + "pct_stereotype": 0.44086021505376344, + "pct_stereotype_stderr": 0.051762678118979284 + }, + "hendrycksTest-philosophy": { + "acc": 0.22508038585209003, + "acc_stderr": 0.023720088516179027, + "acc_norm": 0.2958199356913183, + "acc_norm_stderr": 0.025922371788818784 + }, + "hendrycksTest-virology": { + "acc": 0.1927710843373494, + "acc_stderr": 0.030709824050565274, + "acc_norm": 0.22289156626506024, + "acc_norm_stderr": 0.03240004825594688 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.21761658031088082, + "acc_stderr": 0.029778663037752964, + "acc_norm": 0.24352331606217617, + "acc_norm_stderr": 0.030975436386845436 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 5.912443693693693, + "likelihood_difference_stderr": 0.7158104771703969, + "pct_stereotype": 0.5225225225225225, + "pct_stereotype_stderr": 0.04762473917649626 + }, + "arc_challenge": { + "acc": 0.21416382252559726, + "acc_stderr": 0.011988383205966508, + "acc_norm": 0.24658703071672355, + "acc_norm_stderr": 0.01259572626879014 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.17575757575757575, + "acc_stderr": 0.02972094300622445, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.030117688929503585 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.18, + "acc_stderr": 0.03861229196653695, + "acc_norm": 0.17, + "acc_norm_stderr": 0.03775251680686371 + }, + "piqa": { + "acc": 0.5255712731229597, + "acc_stderr": 0.011650557844573573, + "acc_norm": 0.5217627856365615, + "acc_norm_stderr": 0.011654768618560074 + }, + "hendrycksTest-anatomy": { + "acc": 0.21481481481481482, + "acc_stderr": 0.03547854198560823, + "acc_norm": 0.2962962962962963, + "acc_norm_stderr": 0.03944624162501117 + }, + "hendrycksTest-professional_law": { + "acc": 0.23272490221642764, + "acc_stderr": 0.010792595553888486, + "acc_norm": 0.24315514993481094, + "acc_norm_stderr": 0.010956556654417358 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 6.980769230769231, + "likelihood_difference_stderr": 1.9247068413265722, + "pct_stereotype": 0.38461538461538464, + "pct_stereotype_stderr": 0.1404416814115811 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 6.295454545454546, + "likelihood_difference_stderr": 3.1838670351509126, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 14.890796703296703, + "likelihood_difference_stderr": 1.0225529329727918, + "pct_stereotype": 0.8021978021978022, + "pct_stereotype_stderr": 0.04198895203196222 + }, + "hendrycksTest-computer_security": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.18907563025210083, + "acc_stderr": 0.02543511943810536, + "acc_norm": 0.2815126050420168, + "acc_norm_stderr": 0.029213549414372163 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.20967741935483872, + "acc_stderr": 0.02315787934908352, + "acc_norm": 0.22258064516129034, + "acc_norm_stderr": 0.02366421667164251 + }, + "hendrycksTest-management": { + "acc": 0.2524271844660194, + "acc_stderr": 0.04301250399690879, + "acc_norm": 0.2912621359223301, + "acc_norm_stderr": 0.044986763205729245 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 13.08430325255102, + "likelihood_difference_stderr": 0.865918639747448, + "pct_stereotype": 0.40816326530612246, + "pct_stereotype_stderr": 0.03519659177561531 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "acc_norm": 0.33, + "acc_norm_stderr": 0.04725815626252606 + }, + "crows_pairs_french": { + "likelihood_difference": 10.008921250745379, + "likelihood_difference_stderr": 0.23452315861799014, + "pct_stereotype": 0.569469290399523, + "pct_stereotype_stderr": 0.012094842430543595 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "hendrycksTest-global_facts": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "acc_norm": 0.27, + "acc_norm_stderr": 0.044619604333847394 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 14.821022727272727, + "likelihood_difference_stderr": 1.4754485302166234, + "pct_stereotype": 0.3484848484848485, + "pct_stereotype_stderr": 0.0591013677911929 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.2315270935960591, + "acc_stderr": 0.029678333141444437, + "acc_norm": 0.22660098522167488, + "acc_norm_stderr": 0.029454863835292975 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2503192848020434, + "acc_stderr": 0.01549108895149458, + "acc_norm": 0.26436781609195403, + "acc_norm_stderr": 0.015769984840690518 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.24074074074074073, + "acc_stderr": 0.026067159222275794, + "acc_norm": 0.2814814814814815, + "acc_norm_stderr": 0.027420019350945263 + }, + "logiqa": { + "acc": 0.22887864823348694, + "acc_stderr": 0.01647810727631327, + "acc_norm": 0.24731182795698925, + "acc_norm_stderr": 0.016922842446712393 + }, + "hendrycksTest-nutrition": { + "acc": 0.17973856209150327, + "acc_stderr": 0.021986032182064148, + "acc_norm": 0.238562091503268, + "acc_norm_stderr": 0.024404394928087877 + }, + "hendrycksTest-astronomy": { + "acc": 0.18421052631578946, + "acc_stderr": 0.031546980450822305, + "acc_norm": 0.23684210526315788, + "acc_norm_stderr": 0.03459777606810534 + }, + "hendrycksTest-world_religions": { + "acc": 0.19298245614035087, + "acc_stderr": 0.03026745755489847, + "acc_norm": 0.23391812865497075, + "acc_norm_stderr": 0.03246721765117825 + }, + "winogrande": { + "acc": 0.4988161010260458, + "acc_stderr": 0.014052446290529012 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.25504587155963304, + "acc_stderr": 0.018688500856535853, + "acc_norm": 0.24587155963302754, + "acc_norm_stderr": 0.018461940968708457 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.1574074074074074, + "acc_stderr": 0.03520703990517965, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.042365112580946315 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "hendrycksTest-machine_learning": { + "acc": 0.3125, + "acc_stderr": 0.043994650575715215, + "acc_norm": 0.30357142857142855, + "acc_norm_stderr": 0.04364226155841044 + }, + "hendrycksTest-prehistory": { + "acc": 0.23148148148148148, + "acc_stderr": 0.02346842983245115, + "acc_norm": 0.20679012345679013, + "acc_norm_stderr": 0.022535006705942818 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.26, + "acc_stderr": 0.04408440022768079, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036624 + }, + "sciq": { + "acc": 0.194, + "acc_stderr": 0.012510816141264368, + "acc_norm": 0.216, + "acc_norm_stderr": 0.013019735539307789 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.8372252747252746, + "likelihood_difference_stderr": 0.6727209631618413, + "pct_stereotype": 0.6593406593406593, + "pct_stereotype_stderr": 0.049956709512768704 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 12.433695652173913, + "likelihood_difference_stderr": 0.9697744506204069, + "pct_stereotype": 0.6086956521739131, + "pct_stereotype_stderr": 0.045709346351117126 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 5.949495570866142, + "likelihood_difference_stderr": 0.36479478018512124, + "pct_stereotype": 0.36023622047244097, + "pct_stereotype_stderr": 0.02132060545173939 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816506 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 10.08498023715415, + "likelihood_difference_stderr": 0.5574722426656213, + "pct_stereotype": 0.5652173913043478, + "pct_stereotype_stderr": 0.03122795678881643 + }, + "hendrycksTest-business_ethics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2595419847328244, + "acc_stderr": 0.03844876139785271, + "acc_norm": 0.2595419847328244, + "acc_norm_stderr": 0.03844876139785271 + }, + "crows_pairs_english": { + "likelihood_difference": 5.816375968992248, + "likelihood_difference_stderr": 0.20056629856250444, + "pct_stereotype": 0.4561717352415027, + "pct_stereotype_stderr": 0.012166287275376293 + }, + "hendrycksTest-marketing": { + "acc": 0.23504273504273504, + "acc_stderr": 0.027778835904935437, + "acc_norm": 0.2564102564102564, + "acc_norm_stderr": 0.028605953702004243 + }, + "hendrycksTest-security_studies": { + "acc": 0.3142857142857143, + "acc_stderr": 0.02971932942241746, + "acc_norm": 0.19183673469387755, + "acc_norm_stderr": 0.025206963154225378 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.25462962962962965, + "acc_stderr": 0.029711275860005354, + "acc_norm": 0.2916666666666667, + "acc_norm_stderr": 0.030998666304560534 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.1656441717791411, + "acc_stderr": 0.029208296231259104, + "acc_norm": 0.2392638036809816, + "acc_norm_stderr": 0.033519538795212696 + }, + "hendrycksTest-public_relations": { + "acc": 0.2545454545454545, + "acc_stderr": 0.041723430387053825, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.03694284335337798 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 6.445746527777778, + "likelihood_difference_stderr": 0.5390301887250009, + "pct_stereotype": 0.44907407407407407, + "pct_stereotype_stderr": 0.03392238405321617 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.17721518987341772, + "acc_stderr": 0.024856364184503217, + "acc_norm": 0.21518987341772153, + "acc_norm_stderr": 0.02675082699467616 + }, + "lambada_openai": { + "ppl": 3684074.4512471026, + "ppl_stderr": 360109.5301043941, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "hendrycksTest-college_medicine": { + "acc": 0.20809248554913296, + "acc_stderr": 0.03095289021774988, + "acc_norm": 0.2658959537572254, + "acc_norm_stderr": 0.03368762932259431 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 5.0119140625, + "likelihood_difference_stderr": 0.5134294888398998, + "pct_stereotype": 0.49375, + "pct_stereotype_stderr": 0.027992438382232313 + }, + "crows_pairs_french_age": { + "likelihood_difference": 5.868055555555555, + "likelihood_difference_stderr": 0.7717131357071738, + "pct_stereotype": 0.6444444444444445, + "pct_stereotype_stderr": 0.05074011803597719 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.23737373737373738, + "acc_stderr": 0.03031371053819888, + "acc_norm": 0.2727272727272727, + "acc_norm_stderr": 0.03173071239071724 + }, + "arc_easy": { + "acc": 0.2718855218855219, + "acc_stderr": 0.009129795867310499, + "acc_norm": 0.2516835016835017, + "acc_norm_stderr": 0.008905088235948768 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 8.875135869565218, + "likelihood_difference_stderr": 0.37474334360732336, + "pct_stereotype": 0.6434782608695652, + "pct_stereotype_stderr": 0.022356489247084357 + }, + "hendrycksTest-formal_logic": { + "acc": 0.24603174603174602, + "acc_stderr": 0.03852273364924318, + "acc_norm": 0.23809523809523808, + "acc_norm_stderr": 0.03809523809523809 + }, + "wsc": { + "acc": 0.5961538461538461, + "acc_stderr": 0.048346889526540184 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.23410404624277456, + "acc_stderr": 0.022797110278071138, + "acc_norm": 0.21098265895953758, + "acc_norm_stderr": 0.021966309947043117 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 7.704244548286605, + "likelihood_difference_stderr": 0.45459318084053474, + "pct_stereotype": 0.5295950155763239, + "pct_stereotype_stderr": 0.027901844420051173 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2553191489361702, + "acc_stderr": 0.0285048564705142, + "acc_norm": 0.25957446808510637, + "acc_norm_stderr": 0.02865917937429232 + }, + "hendrycksTest-international_law": { + "acc": 0.08264462809917356, + "acc_stderr": 0.02513538235660422, + "acc_norm": 0.2066115702479339, + "acc_norm_stderr": 0.03695980128098824 + }, + "hendrycksTest-college_biology": { + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "acc_norm": 0.2638888888888889, + "acc_norm_stderr": 0.03685651095897532 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.22058823529411764, + "acc_stderr": 0.01677467236546854, + "acc_norm": 0.23366013071895425, + "acc_norm_stderr": 0.017119158496044503 + }, + "hendrycksTest-econometrics": { + "acc": 0.2543859649122807, + "acc_stderr": 0.04096985139843669, + "acc_norm": 0.30701754385964913, + "acc_norm_stderr": 0.0433913832257986 + }, + "hendrycksTest-college_physics": { + "acc": 0.23529411764705882, + "acc_stderr": 0.04220773659171452, + "acc_norm": 0.22549019607843138, + "acc_norm_stderr": 0.04158307533083286 + }, + "hendrycksTest-sociology": { + "acc": 0.25870646766169153, + "acc_stderr": 0.030965903123573044, + "acc_norm": 0.2736318407960199, + "acc_norm_stderr": 0.03152439186555402 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.19310344827586207, + "acc_stderr": 0.03289445522127401, + "acc_norm": 0.19310344827586207, + "acc_norm_stderr": 0.03289445522127401 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.18627450980392157, + "acc_stderr": 0.027325470966716326, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.030964517926923393 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.2275132275132275, + "acc_stderr": 0.021591269407823785, + "acc_norm": 0.23544973544973544, + "acc_norm_stderr": 0.021851509822031715 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.24632352941176472, + "acc_stderr": 0.02617343857052, + "acc_norm": 0.27941176470588236, + "acc_norm_stderr": 0.02725720260611495 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 7.943269230769231, + "likelihood_difference_stderr": 1.246261479003386, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.06231481440776789 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 6.254340277777778, + "likelihood_difference_stderr": 0.84909722861177, + "pct_stereotype": 0.5416666666666666, + "pct_stereotype_stderr": 0.05913268547421811 + }, + "hendrycksTest-human_aging": { + "acc": 0.28699551569506726, + "acc_stderr": 0.030360379710291936, + "acc_norm": 0.273542600896861, + "acc_norm_stderr": 0.029918586707798817 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.17, + "acc_stderr": 0.0377525168068637, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036623 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.18867924528301888, + "acc_stderr": 0.024079995130062207, + "acc_norm": 0.2792452830188679, + "acc_norm_stderr": 0.027611163402399715 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 6.05641447368421, + "likelihood_difference_stderr": 0.5511819524754867, + "pct_stereotype": 0.46842105263157896, + "pct_stereotype_stderr": 0.03629703808831611 + } + }, + "versions": { + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-us_foreign_policy": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "crows_pairs_english_religion": 0, + "arc_challenge": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-college_mathematics": 0, + "piqa": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-professional_law": 0, + "crows_pairs_french_autre": 0, + "crows_pairs_english_autre": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-management": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-high_school_computer_science": 0, + "crows_pairs_french": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-global_facts": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-high_school_mathematics": 0, + "logiqa": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-world_religions": 0, + "winogrande": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-college_computer_science": 0, + "sciq": 0, + "crows_pairs_english_age": 0, + "crows_pairs_french_religion": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-abstract_algebra": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_english": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-public_relations": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-high_school_world_history": 0, + "lambada_openai": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_english_gender": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-high_school_geography": 0, + "arc_easy": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-formal_logic": 0, + "wsc": 0, + "hendrycksTest-moral_disputes": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-professional_medicine": 0, + "crows_pairs_english_disability": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_english_socioeconomic": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step0", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step1.json b/data/checkpoint_eval/eleutherai_evals/step1.json new file mode 100644 index 0000000000000000000000000000000000000000..fd0948d86a707fa6f242b83f76316ca3d86c2dda --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step1.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-college_mathematics": { + "acc": 0.18, + "acc_stderr": 0.03861229196653695, + "acc_norm": 0.17, + "acc_norm_stderr": 0.03775251680686371 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 6.254340277777778, + "likelihood_difference_stderr": 0.84909722861177, + "pct_stereotype": 0.5416666666666666, + "pct_stereotype_stderr": 0.05913268547421811 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.25504587155963304, + "acc_stderr": 0.018688500856535853, + "acc_norm": 0.24587155963302754, + "acc_norm_stderr": 0.018461940968708457 + }, + "hendrycksTest-security_studies": { + "acc": 0.3142857142857143, + "acc_stderr": 0.02971932942241746, + "acc_norm": 0.19183673469387755, + "acc_norm_stderr": 0.025206963154225378 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.22058823529411764, + "acc_stderr": 0.01677467236546854, + "acc_norm": 0.23366013071895425, + "acc_norm_stderr": 0.017119158496044503 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.18867924528301888, + "acc_stderr": 0.024079995130062207, + "acc_norm": 0.2792452830188679, + "acc_norm_stderr": 0.027611163402399715 + }, + "arc_easy": { + "acc": 0.2718855218855219, + "acc_stderr": 0.009129795867310499, + "acc_norm": 0.2516835016835017, + "acc_norm_stderr": 0.008905088235948768 + }, + "hendrycksTest-public_relations": { + "acc": 0.2545454545454545, + "acc_stderr": 0.041723430387053825, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.03694284335337798 + }, + "hendrycksTest-human_aging": { + "acc": 0.28699551569506726, + "acc_stderr": 0.030360379710291936, + "acc_norm": 0.273542600896861, + "acc_norm_stderr": 0.029918586707798817 + }, + "hendrycksTest-machine_learning": { + "acc": 0.3125, + "acc_stderr": 0.043994650575715215, + "acc_norm": 0.30357142857142855, + "acc_norm_stderr": 0.04364226155841044 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.21761658031088082, + "acc_stderr": 0.029778663037752964, + "acc_norm": 0.24352331606217617, + "acc_norm_stderr": 0.030975436386845436 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 8.875135869565218, + "likelihood_difference_stderr": 0.37474334360732336, + "pct_stereotype": 0.6434782608695652, + "pct_stereotype_stderr": 0.022356489247084357 + }, + "wsc": { + "acc": 0.5961538461538461, + "acc_stderr": 0.048346889526540184 + }, + "piqa": { + "acc": 0.5255712731229597, + "acc_stderr": 0.011650557844573573, + "acc_norm": 0.5217627856365615, + "acc_norm_stderr": 0.011654768618560074 + }, + "hendrycksTest-professional_law": { + "acc": 0.23272490221642764, + "acc_stderr": 0.010792595553888486, + "acc_norm": 0.24315514993481094, + "acc_norm_stderr": 0.010956556654417358 + }, + "sciq": { + "acc": 0.194, + "acc_stderr": 0.012510816141264368, + "acc_norm": 0.216, + "acc_norm_stderr": 0.013019735539307789 + }, + "logiqa": { + "acc": 0.22887864823348694, + "acc_stderr": 0.01647810727631327, + "acc_norm": 0.24731182795698925, + "acc_norm_stderr": 0.016922842446712393 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.21794871794871795, + "acc_stderr": 0.020932445774463196, + "acc_norm": 0.24358974358974358, + "acc_norm_stderr": 0.02176373368417392 + }, + "hendrycksTest-formal_logic": { + "acc": 0.24603174603174602, + "acc_stderr": 0.03852273364924318, + "acc_norm": 0.23809523809523808, + "acc_norm_stderr": 0.03809523809523809 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2503192848020434, + "acc_stderr": 0.01549108895149458, + "acc_norm": 0.26436781609195403, + "acc_norm_stderr": 0.015769984840690518 + }, + "crows_pairs_french": { + "likelihood_difference": 10.008921250745379, + "likelihood_difference_stderr": 0.23452315861799014, + "pct_stereotype": 0.569469290399523, + "pct_stereotype_stderr": 0.012094842430543595 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "hendrycksTest-world_religions": { + "acc": 0.19298245614035087, + "acc_stderr": 0.03026745755489847, + "acc_norm": 0.23391812865497075, + "acc_norm_stderr": 0.03246721765117825 + }, + "crows_pairs_english": { + "likelihood_difference": 5.816375968992248, + "likelihood_difference_stderr": 0.20056629856250444, + "pct_stereotype": 0.4561717352415027, + "pct_stereotype_stderr": 0.012166287275376293 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.2275132275132275, + "acc_stderr": 0.021591269407823785, + "acc_norm": 0.23544973544973544, + "acc_norm_stderr": 0.021851509822031715 + }, + "arc_challenge": { + "acc": 0.21416382252559726, + "acc_stderr": 0.011988383205966508, + "acc_norm": 0.24658703071672355, + "acc_norm_stderr": 0.01259572626879014 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.8372252747252746, + "likelihood_difference_stderr": 0.6727209631618413, + "pct_stereotype": 0.6593406593406593, + "pct_stereotype_stderr": 0.049956709512768704 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.25462962962962965, + "acc_stderr": 0.029711275860005354, + "acc_norm": 0.2916666666666667, + "acc_norm_stderr": 0.030998666304560534 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 13.08430325255102, + "likelihood_difference_stderr": 0.865918639747448, + "pct_stereotype": 0.40816326530612246, + "pct_stereotype_stderr": 0.03519659177561531 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.23737373737373738, + "acc_stderr": 0.03031371053819888, + "acc_norm": 0.2727272727272727, + "acc_norm_stderr": 0.03173071239071724 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.18627450980392157, + "acc_stderr": 0.027325470966716326, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.030964517926923393 + }, + "hendrycksTest-philosophy": { + "acc": 0.22508038585209003, + "acc_stderr": 0.023720088516179027, + "acc_norm": 0.2958199356913183, + "acc_norm_stderr": 0.025922371788818784 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.24632352941176472, + "acc_stderr": 0.02617343857052, + "acc_norm": 0.27941176470588236, + "acc_norm_stderr": 0.02725720260611495 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 5.912443693693693, + "likelihood_difference_stderr": 0.7158104771703969, + "pct_stereotype": 0.5225225225225225, + "pct_stereotype_stderr": 0.04762473917649626 + }, + "hendrycksTest-astronomy": { + "acc": 0.18421052631578946, + "acc_stderr": 0.031546980450822305, + "acc_norm": 0.23684210526315788, + "acc_norm_stderr": 0.03459777606810534 + }, + "hendrycksTest-international_law": { + "acc": 0.08264462809917356, + "acc_stderr": 0.02513538235660422, + "acc_norm": 0.2066115702479339, + "acc_norm_stderr": 0.03695980128098824 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.18543046357615894, + "acc_stderr": 0.031732843842942865, + "acc_norm": 0.31125827814569534, + "acc_norm_stderr": 0.03780445850526733 + }, + "hendrycksTest-management": { + "acc": 0.2524271844660194, + "acc_stderr": 0.04301250399690879, + "acc_norm": 0.2912621359223301, + "acc_norm_stderr": 0.044986763205729245 + }, + "hendrycksTest-prehistory": { + "acc": 0.23148148148148148, + "acc_stderr": 0.02346842983245115, + "acc_norm": 0.20679012345679013, + "acc_norm_stderr": 0.022535006705942818 + }, + "hendrycksTest-anatomy": { + "acc": 0.21481481481481482, + "acc_stderr": 0.03547854198560823, + "acc_norm": 0.2962962962962963, + "acc_norm_stderr": 0.03944624162501117 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.17575757575757575, + "acc_stderr": 0.02972094300622445, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.030117688929503585 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 10.157118055555555, + "likelihood_difference_stderr": 1.3610391208223753, + "pct_stereotype": 0.5138888888888888, + "pct_stereotype_stderr": 0.05931618532716555 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 5.949495570866142, + "likelihood_difference_stderr": 0.36479478018512124, + "pct_stereotype": 0.36023622047244097, + "pct_stereotype_stderr": 0.02132060545173939 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.17721518987341772, + "acc_stderr": 0.024856364184503217, + "acc_norm": 0.21518987341772153, + "acc_norm_stderr": 0.02675082699467616 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 7.704244548286605, + "likelihood_difference_stderr": 0.45459318084053474, + "pct_stereotype": 0.5295950155763239, + "pct_stereotype_stderr": 0.027901844420051173 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.18907563025210083, + "acc_stderr": 0.02543511943810536, + "acc_norm": 0.2815126050420168, + "acc_norm_stderr": 0.029213549414372163 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.1574074074074074, + "acc_stderr": 0.03520703990517965, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.042365112580946315 + }, + "hendrycksTest-nutrition": { + "acc": 0.17973856209150327, + "acc_stderr": 0.021986032182064148, + "acc_norm": 0.238562091503268, + "acc_norm_stderr": 0.024404394928087877 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816506 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "acc_norm": 0.33, + "acc_norm_stderr": 0.04725815626252606 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 14.821022727272727, + "likelihood_difference_stderr": 1.4754485302166234, + "pct_stereotype": 0.3484848484848485, + "pct_stereotype_stderr": 0.0591013677911929 + }, + "hendrycksTest-sociology": { + "acc": 0.25870646766169153, + "acc_stderr": 0.030965903123573044, + "acc_norm": 0.2736318407960199, + "acc_norm_stderr": 0.03152439186555402 + }, + "hendrycksTest-college_physics": { + "acc": 0.23529411764705882, + "acc_stderr": 0.04220773659171452, + "acc_norm": 0.22549019607843138, + "acc_norm_stderr": 0.04158307533083286 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.2315270935960591, + "acc_stderr": 0.029678333141444437, + "acc_norm": 0.22660098522167488, + "acc_norm_stderr": 0.029454863835292975 + }, + "winogrande": { + "acc": 0.4988161010260458, + "acc_stderr": 0.014052446290529012 + }, + "hendrycksTest-college_medicine": { + "acc": 0.20809248554913296, + "acc_stderr": 0.03095289021774988, + "acc_norm": 0.2658959537572254, + "acc_norm_stderr": 0.03368762932259431 + }, + "hendrycksTest-global_facts": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "acc_norm": 0.27, + "acc_norm_stderr": 0.044619604333847394 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.23410404624277456, + "acc_stderr": 0.022797110278071138, + "acc_norm": 0.21098265895953758, + "acc_norm_stderr": 0.021966309947043117 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 7.943269230769231, + "likelihood_difference_stderr": 1.246261479003386, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.06231481440776789 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 6.05641447368421, + "likelihood_difference_stderr": 0.5511819524754867, + "pct_stereotype": 0.46842105263157896, + "pct_stereotype_stderr": 0.03629703808831611 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 6.445746527777778, + "likelihood_difference_stderr": 0.5390301887250009, + "pct_stereotype": 0.44907407407407407, + "pct_stereotype_stderr": 0.03392238405321617 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 6.295454545454546, + "likelihood_difference_stderr": 3.1838670351509126, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.17, + "acc_stderr": 0.0377525168068637, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036623 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 5.0119140625, + "likelihood_difference_stderr": 0.5134294888398998, + "pct_stereotype": 0.49375, + "pct_stereotype_stderr": 0.027992438382232313 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.1656441717791411, + "acc_stderr": 0.029208296231259104, + "acc_norm": 0.2392638036809816, + "acc_norm_stderr": 0.033519538795212696 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 6.980769230769231, + "likelihood_difference_stderr": 1.9247068413265722, + "pct_stereotype": 0.38461538461538464, + "pct_stereotype_stderr": 0.1404416814115811 + }, + "hendrycksTest-virology": { + "acc": 0.1927710843373494, + "acc_stderr": 0.030709824050565274, + "acc_norm": 0.22289156626506024, + "acc_norm_stderr": 0.03240004825594688 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.20967741935483872, + "acc_stderr": 0.02315787934908352, + "acc_norm": 0.22258064516129034, + "acc_norm_stderr": 0.02366421667164251 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 14.890796703296703, + "likelihood_difference_stderr": 1.0225529329727918, + "pct_stereotype": 0.8021978021978022, + "pct_stereotype_stderr": 0.04198895203196222 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2595419847328244, + "acc_stderr": 0.03844876139785271, + "acc_norm": 0.2595419847328244, + "acc_norm_stderr": 0.03844876139785271 + }, + "crows_pairs_french_age": { + "likelihood_difference": 5.868055555555555, + "likelihood_difference_stderr": 0.7717131357071738, + "pct_stereotype": 0.6444444444444445, + "pct_stereotype_stderr": 0.05074011803597719 + }, + "lambada_openai": { + "ppl": 3684074.4512471026, + "ppl_stderr": 360109.5301043941, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.24074074074074073, + "acc_stderr": 0.026067159222275794, + "acc_norm": 0.2814814814814815, + "acc_norm_stderr": 0.027420019350945263 + }, + "hendrycksTest-business_ethics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-marketing": { + "acc": 0.23504273504273504, + "acc_stderr": 0.027778835904935437, + "acc_norm": 0.2564102564102564, + "acc_norm_stderr": 0.028605953702004243 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "hendrycksTest-econometrics": { + "acc": 0.2543859649122807, + "acc_stderr": 0.04096985139843669, + "acc_norm": 0.30701754385964913, + "acc_norm_stderr": 0.0433913832257986 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2553191489361702, + "acc_stderr": 0.0285048564705142, + "acc_norm": 0.25957446808510637, + "acc_norm_stderr": 0.02865917937429232 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.844758064516129, + "likelihood_difference_stderr": 0.7849858741881288, + "pct_stereotype": 0.44086021505376344, + "pct_stereotype_stderr": 0.051762678118979284 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 10.08498023715415, + "likelihood_difference_stderr": 0.5574722426656213, + "pct_stereotype": 0.5652173913043478, + "pct_stereotype_stderr": 0.03122795678881643 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 12.433695652173913, + "likelihood_difference_stderr": 0.9697744506204069, + "pct_stereotype": 0.6086956521739131, + "pct_stereotype_stderr": 0.045709346351117126 + }, + "hendrycksTest-college_biology": { + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "acc_norm": 0.2638888888888889, + "acc_norm_stderr": 0.03685651095897532 + }, + "hendrycksTest-computer_security": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.19310344827586207, + "acc_stderr": 0.03289445522127401, + "acc_norm": 0.19310344827586207, + "acc_norm_stderr": 0.03289445522127401 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.26, + "acc_stderr": 0.04408440022768079, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036624 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2765957446808511, + "acc_stderr": 0.026684564340460997, + "acc_norm": 0.2624113475177305, + "acc_norm_stderr": 0.026244920349843007 + } + }, + "versions": { + "hendrycksTest-college_mathematics": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-clinical_knowledge": 0, + "arc_easy": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "crows_pairs_french_race_color": 0, + "wsc": 0, + "piqa": 0, + "hendrycksTest-professional_law": 0, + "sciq": 0, + "logiqa": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-miscellaneous": 0, + "crows_pairs_french": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_english": 0, + "hendrycksTest-elementary_mathematics": 0, + "arc_challenge": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-high_school_statistics": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-professional_medicine": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-management": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-high_school_european_history": 0, + "crows_pairs_french_physical_appearance": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-high_school_world_history": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-high_school_computer_science": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-high_school_chemistry": 0, + "winogrande": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-moral_disputes": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-moral_scenarios": 0, + "crows_pairs_english_socioeconomic": 0, + "crows_pairs_english_nationality": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-college_chemistry": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-logical_fallacies": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-high_school_biology": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_french_age": 0, + "lambada_openai": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-conceptual_physics": 0, + "crows_pairs_english_sexual_orientation": 0, + "crows_pairs_french_nationality": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-professional_accounting": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step1", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step1000.json b/data/checkpoint_eval/eleutherai_evals/step1000.json new file mode 100644 index 0000000000000000000000000000000000000000..7dcd97e9375c6fb994d4f72ca064813be510e7a6 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step1000.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-astronomy": { + "acc": 0.20394736842105263, + "acc_stderr": 0.032790004063100495, + "acc_norm": 0.28289473684210525, + "acc_norm_stderr": 0.03665349695640767 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.1388888888888889, + "acc_stderr": 0.03343270062869621, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.04236511258094631 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.29770992366412213, + "acc_stderr": 0.040103589424622034, + "acc_norm": 0.25190839694656486, + "acc_norm_stderr": 0.03807387116306086 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.21631205673758866, + "acc_stderr": 0.024561720560562793, + "acc_norm": 0.22340425531914893, + "acc_norm_stderr": 0.02484792135806396 + }, + "hendrycksTest-sociology": { + "acc": 0.23383084577114427, + "acc_stderr": 0.029929415408348377, + "acc_norm": 0.19900497512437812, + "acc_norm_stderr": 0.02823136509275841 + }, + "hendrycksTest-machine_learning": { + "acc": 0.24107142857142858, + "acc_stderr": 0.04059867246952685, + "acc_norm": 0.2767857142857143, + "acc_norm_stderr": 0.04246624336697624 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.24770642201834864, + "acc_stderr": 0.018508143602547822, + "acc_norm": 0.24220183486238533, + "acc_norm_stderr": 0.01836817630659862 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.19327731092436976, + "acc_stderr": 0.025649470265889186, + "acc_norm": 0.2815126050420168, + "acc_norm_stderr": 0.02921354941437216 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.863888888888889, + "likelihood_difference_stderr": 0.43040277080568906, + "pct_stereotype": 0.3333333333333333, + "pct_stereotype_stderr": 0.04996877926639073 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.300341796875, + "likelihood_difference_stderr": 0.30728727580611953, + "pct_stereotype": 0.5625, + "pct_stereotype_stderr": 0.02777505646718807 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.19704433497536947, + "acc_stderr": 0.02798672466673622, + "acc_norm": 0.2315270935960591, + "acc_norm_stderr": 0.029678333141444465 + }, + "hendrycksTest-business_ethics": { + "acc": 0.34, + "acc_stderr": 0.04760952285695235, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206845 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.53125, + "likelihood_difference_stderr": 0.22379421806000957, + "pct_stereotype": 0.49221183800623053, + "pct_stereotype_stderr": 0.02794745876935634 + }, + "hendrycksTest-college_physics": { + "acc": 0.16666666666666666, + "acc_stderr": 0.03708284662416542, + "acc_norm": 0.19607843137254902, + "acc_norm_stderr": 0.03950581861179963 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.12, + "acc_stderr": 0.03265986323710906, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-nutrition": { + "acc": 0.20588235294117646, + "acc_stderr": 0.023152722439402307, + "acc_norm": 0.31699346405228757, + "acc_norm_stderr": 0.02664327847450875 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.24673202614379086, + "acc_stderr": 0.0174408203674025, + "acc_norm": 0.2581699346405229, + "acc_norm_stderr": 0.017704531653250068 + }, + "hendrycksTest-global_facts": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036623 + }, + "crows_pairs_french": { + "likelihood_difference": 6.4261003652355395, + "likelihood_difference_stderr": 0.15537974449952036, + "pct_stereotype": 0.456768038163387, + "pct_stereotype_stderr": 0.012167560197793081 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.28, + "acc_stderr": 0.04512608598542127, + "acc_norm": 0.26, + "acc_norm_stderr": 0.04408440022768081 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.23232323232323232, + "acc_stderr": 0.030088629490217483, + "acc_norm": 0.29292929292929293, + "acc_norm_stderr": 0.032424979581788166 + }, + "hendrycksTest-prehistory": { + "acc": 0.3055555555555556, + "acc_stderr": 0.02563082497562135, + "acc_norm": 0.2345679012345679, + "acc_norm_stderr": 0.023576881744005712 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.18787878787878787, + "acc_stderr": 0.03050193405942914, + "acc_norm": 0.3212121212121212, + "acc_norm_stderr": 0.03646204963253812 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.25384615384615383, + "acc_stderr": 0.022066054378726257, + "acc_norm": 0.2512820512820513, + "acc_norm_stderr": 0.021992016662370564 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.2185430463576159, + "acc_stderr": 0.033742355504256936, + "acc_norm": 0.26490066225165565, + "acc_norm_stderr": 0.03603038545360384 + }, + "hendrycksTest-college_medicine": { + "acc": 0.23699421965317918, + "acc_stderr": 0.03242414757483098, + "acc_norm": 0.26011560693641617, + "acc_norm_stderr": 0.03345036916788991 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.2398843930635838, + "acc_stderr": 0.022989592543123567, + "acc_norm": 0.2514450867052023, + "acc_norm_stderr": 0.023357365785874037 + }, + "logiqa": { + "acc": 0.20276497695852536, + "acc_stderr": 0.015770046635584567, + "acc_norm": 0.2350230414746544, + "acc_norm_stderr": 0.016631166823890972 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.22699386503067484, + "acc_stderr": 0.03291099578615769, + "acc_norm": 0.26993865030674846, + "acc_norm_stderr": 0.03487825168497892 + }, + "hendrycksTest-anatomy": { + "acc": 0.2740740740740741, + "acc_stderr": 0.03853254836552003, + "acc_norm": 0.28888888888888886, + "acc_norm_stderr": 0.0391545063041425 + }, + "hendrycksTest-college_biology": { + "acc": 0.3055555555555556, + "acc_stderr": 0.03852084696008534, + "acc_norm": 0.25, + "acc_norm_stderr": 0.03621034121889507 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.24, + "acc_stderr": 0.04292346959909283, + "acc_norm": 0.21, + "acc_norm_stderr": 0.040936018074033256 + }, + "hendrycksTest-international_law": { + "acc": 0.09917355371900827, + "acc_stderr": 0.027285246312758957, + "acc_norm": 0.2727272727272727, + "acc_norm_stderr": 0.04065578140908705 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 7.329545454545454, + "likelihood_difference_stderr": 2.7739733833639377, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.8755228838582676, + "likelihood_difference_stderr": 0.18603255188854018, + "pct_stereotype": 0.39763779527559057, + "pct_stereotype_stderr": 0.021735453502117665 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.24150943396226415, + "acc_stderr": 0.026341480371118355, + "acc_norm": 0.30566037735849055, + "acc_norm_stderr": 0.028353298073322666 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.27205882352941174, + "acc_stderr": 0.027033041151681456, + "acc_norm": 0.28308823529411764, + "acc_norm_stderr": 0.02736586113151381 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 14.403846153846153, + "likelihood_difference_stderr": 1.0171287098668624, + "pct_stereotype": 0.8571428571428571, + "pct_stereotype_stderr": 0.036885555678165906 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.9060461956521735, + "likelihood_difference_stderr": 0.25578693071097974, + "pct_stereotype": 0.45652173913043476, + "pct_stereotype_stderr": 0.023249599562309698 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.23148148148148148, + "acc_stderr": 0.02876511171804697, + "acc_norm": 0.2916666666666667, + "acc_norm_stderr": 0.030998666304560517 + }, + "hendrycksTest-human_aging": { + "acc": 0.34080717488789236, + "acc_stderr": 0.03181149747055359, + "acc_norm": 0.26905829596412556, + "acc_norm_stderr": 0.029763779406874972 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.8465711805555554, + "likelihood_difference_stderr": 0.4255545660702197, + "pct_stereotype": 0.5972222222222222, + "pct_stereotype_stderr": 0.05820650942569532 + }, + "hendrycksTest-formal_logic": { + "acc": 0.30952380952380953, + "acc_stderr": 0.04134913018303316, + "acc_norm": 0.23809523809523808, + "acc_norm_stderr": 0.03809523809523812 + }, + "hendrycksTest-professional_law": { + "acc": 0.23663624511082137, + "acc_stderr": 0.01085513735157273, + "acc_norm": 0.25358539765319427, + "acc_norm_stderr": 0.011111715336101148 + }, + "arc_easy": { + "acc": 0.3085016835016835, + "acc_stderr": 0.009477472342978136, + "acc_norm": 0.3005050505050505, + "acc_norm_stderr": 0.009407763090599316 + }, + "hendrycksTest-econometrics": { + "acc": 0.2719298245614035, + "acc_stderr": 0.041857744240220575, + "acc_norm": 0.2631578947368421, + "acc_norm_stderr": 0.04142439719489362 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.488911290322581, + "likelihood_difference_stderr": 0.6831591300050538, + "pct_stereotype": 0.7096774193548387, + "pct_stereotype_stderr": 0.047323514218241214 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 6.357638888888889, + "likelihood_difference_stderr": 0.785656073241832, + "pct_stereotype": 0.4722222222222222, + "pct_stereotype_stderr": 0.05924743948371487 + }, + "hendrycksTest-public_relations": { + "acc": 0.2545454545454545, + "acc_stderr": 0.04172343038705383, + "acc_norm": 0.12727272727272726, + "acc_norm_stderr": 0.031922265124685704 + }, + "wsc": { + "acc": 0.5576923076923077, + "acc_stderr": 0.04893740777701 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 5.1298076923076925, + "likelihood_difference_stderr": 1.1002089858855335, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.14390989949130545 + }, + "sciq": { + "acc": 0.452, + "acc_stderr": 0.015746235865880677, + "acc_norm": 0.424, + "acc_norm_stderr": 0.015635487471405182 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.4934752747252746, + "likelihood_difference_stderr": 0.40548101485073207, + "pct_stereotype": 0.3516483516483517, + "pct_stereotype_stderr": 0.050331323186278906 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.24, + "acc_stderr": 0.04292346959909283, + "acc_norm": 0.21, + "acc_norm_stderr": 0.040936018074033256 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.24050632911392406, + "acc_stderr": 0.027820781981149675, + "acc_norm": 0.28270042194092826, + "acc_norm_stderr": 0.029312814153955924 + }, + "lambada_openai": { + "ppl": 4465.3093044480365, + "ppl_stderr": 200.19599611051817, + "acc": 0.050067921599068504, + "acc_stderr": 0.0030383523145974317 + }, + "hendrycksTest-virology": { + "acc": 0.25903614457831325, + "acc_stderr": 0.034106466140718564, + "acc_norm": 0.27710843373493976, + "acc_norm_stderr": 0.03484331592680588 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.27262569832402234, + "acc_stderr": 0.014893391735249588, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.26382978723404255, + "acc_stderr": 0.028809989854102963, + "acc_norm": 0.2, + "acc_norm_stderr": 0.0261488180184245 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.25925925925925924, + "acc_stderr": 0.02256989707491842, + "acc_norm": 0.2804232804232804, + "acc_norm_stderr": 0.02313528797432562 + }, + "winogrande": { + "acc": 0.5114443567482242, + "acc_stderr": 0.014048804199859325 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 4.189163773148148, + "likelihood_difference_stderr": 0.31534069144605886, + "pct_stereotype": 0.37037037037037035, + "pct_stereotype_stderr": 0.03293377139415191 + }, + "arc_challenge": { + "acc": 0.1697952218430034, + "acc_stderr": 0.01097177515778422, + "acc_norm": 0.21160409556313994, + "acc_norm_stderr": 0.011935916358632856 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.22549019607843138, + "acc_stderr": 0.02933116229425173, + "acc_norm": 0.2549019607843137, + "acc_norm_stderr": 0.030587591351604257 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.309615384615385, + "likelihood_difference_stderr": 0.6646931122976717, + "pct_stereotype": 0.5076923076923077, + "pct_stereotype_stderr": 0.062492603112584276 + }, + "hendrycksTest-security_studies": { + "acc": 0.3183673469387755, + "acc_stderr": 0.029822533793982055, + "acc_norm": 0.17959183673469387, + "acc_norm_stderr": 0.024573293589585637 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.28, + "acc_stderr": 0.045126085985421276, + "acc_norm": 0.35, + "acc_norm_stderr": 0.0479372485441102 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 5.877909757653061, + "likelihood_difference_stderr": 0.43880029304541995, + "pct_stereotype": 0.4897959183673469, + "pct_stereotype_stderr": 0.03579828650232779 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 6.877445652173913, + "likelihood_difference_stderr": 0.5373839838941108, + "pct_stereotype": 0.5304347826086957, + "pct_stereotype_stderr": 0.046742456376794195 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.24393358876117496, + "acc_stderr": 0.015357212665829484, + "acc_norm": 0.25798212005108556, + "acc_norm_stderr": 0.01564583018834895 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.1962962962962963, + "acc_stderr": 0.024217421327417152, + "acc_norm": 0.24444444444444444, + "acc_norm_stderr": 0.02620276653465215 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.816300675675676, + "likelihood_difference_stderr": 0.4939709971871691, + "pct_stereotype": 0.6126126126126126, + "pct_stereotype_stderr": 0.0464482507235508 + }, + "crows_pairs_english": { + "likelihood_difference": 3.9560878801431127, + "likelihood_difference_stderr": 0.11739151165792377, + "pct_stereotype": 0.5002981514609421, + "pct_stereotype_stderr": 0.01221329704726544 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2689655172413793, + "acc_stderr": 0.036951833116502325, + "acc_norm": 0.23448275862068965, + "acc_norm_stderr": 0.035306258743465914 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 9.315958498023715, + "likelihood_difference_stderr": 0.4249948667508893, + "pct_stereotype": 0.26877470355731226, + "pct_stereotype_stderr": 0.0279266941681729 + }, + "piqa": { + "acc": 0.5603917301414582, + "acc_stderr": 0.011580417248656579, + "acc_norm": 0.5391730141458106, + "acc_norm_stderr": 0.011629966056957104 + }, + "hendrycksTest-marketing": { + "acc": 0.2692307692307692, + "acc_stderr": 0.029058588303748845, + "acc_norm": 0.3034188034188034, + "acc_norm_stderr": 0.030118210106942635 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.27461139896373055, + "acc_stderr": 0.03221024508041156, + "acc_norm": 0.26424870466321243, + "acc_norm_stderr": 0.03182155050916647 + }, + "hendrycksTest-computer_security": { + "acc": 0.24, + "acc_stderr": 0.04292346959909283, + "acc_norm": 0.32, + "acc_norm_stderr": 0.046882617226215034 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 7.460700757575758, + "likelihood_difference_stderr": 0.7513634625351047, + "pct_stereotype": 0.36363636363636365, + "pct_stereotype_stderr": 0.05966637484671758 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.2032258064516129, + "acc_stderr": 0.022891687984554963, + "acc_norm": 0.26129032258064516, + "acc_norm_stderr": 0.02499305339776483 + }, + "hendrycksTest-management": { + "acc": 0.2524271844660194, + "acc_stderr": 0.04301250399690878, + "acc_norm": 0.2621359223300971, + "acc_norm_stderr": 0.04354631077260595 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 4.436513157894737, + "likelihood_difference_stderr": 0.2958745780592147, + "pct_stereotype": 0.6842105263157895, + "pct_stereotype_stderr": 0.033811372338927476 + }, + "hendrycksTest-world_religions": { + "acc": 0.23391812865497075, + "acc_stderr": 0.03246721765117827, + "acc_norm": 0.29239766081871343, + "acc_norm_stderr": 0.03488647713457921 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "acc_norm": 0.22, + "acc_norm_stderr": 0.041633319989322695 + }, + "hendrycksTest-philosophy": { + "acc": 0.21221864951768488, + "acc_stderr": 0.023222756797435126, + "acc_norm": 0.2347266881028939, + "acc_norm_stderr": 0.024071805887677045 + } + }, + "versions": { + "hendrycksTest-astronomy": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-high_school_microeconomics": 0, + "crows_pairs_french_age": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-business_ethics": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-global_facts": 0, + "crows_pairs_french": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-moral_disputes": 0, + "logiqa": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-international_law": 0, + "crows_pairs_english_autre": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-professional_medicine": 0, + "crows_pairs_french_sexual_orientation": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-human_aging": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-professional_law": 0, + "arc_easy": 0, + "hendrycksTest-econometrics": 0, + "crows_pairs_english_sexual_orientation": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-public_relations": 0, + "wsc": 0, + "crows_pairs_french_autre": 0, + "sciq": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-high_school_world_history": 0, + "lambada_openai": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-elementary_mathematics": 0, + "winogrande": 0, + "crows_pairs_english_nationality": 0, + "arc_challenge": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-college_chemistry": 0, + "crows_pairs_french_socioeconomic": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-college_computer_science": 0, + "crows_pairs_english_religion": 0, + "crows_pairs_english": 0, + "hendrycksTest-electrical_engineering": 0, + "crows_pairs_french_nationality": 0, + "piqa": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-computer_security": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-management": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-philosophy": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step1000", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step103000.json b/data/checkpoint_eval/eleutherai_evals/step103000.json new file mode 100644 index 0000000000000000000000000000000000000000..90261754bf2c4982cff3556b7d2394b840d7dc66 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step103000.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.517809139784946, + "likelihood_difference_stderr": 0.5053952638835562, + "pct_stereotype": 0.8172043010752689, + "pct_stereotype_stderr": 0.040295300106155174 + }, + "hendrycksTest-human_aging": { + "acc": 0.3004484304932735, + "acc_stderr": 0.03076935200822915, + "acc_norm": 0.29596412556053814, + "acc_norm_stderr": 0.030636591348699796 + }, + "hendrycksTest-college_medicine": { + "acc": 0.19653179190751446, + "acc_stderr": 0.030299574664788147, + "acc_norm": 0.26011560693641617, + "acc_norm_stderr": 0.033450369167889925 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.677513586956522, + "likelihood_difference_stderr": 0.2551270290244533, + "pct_stereotype": 0.40217391304347827, + "pct_stereotype_stderr": 0.022886956104263133 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.5828269675925926, + "likelihood_difference_stderr": 0.2800526171673634, + "pct_stereotype": 0.38425925925925924, + "pct_stereotype_stderr": 0.03317354514310742 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.892578125, + "likelihood_difference_stderr": 0.45171267642276286, + "pct_stereotype": 0.6111111111111112, + "pct_stereotype_stderr": 0.057855371034784615 + }, + "lambada_openai": { + "ppl": 125.56792285427366, + "ppl_stderr": 5.322330936055688, + "acc": 0.22608189404230544, + "acc_stderr": 0.005827636676144432 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2748091603053435, + "acc_stderr": 0.03915345408847837, + "acc_norm": 0.2595419847328244, + "acc_norm_stderr": 0.03844876139785271 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.28308823529411764, + "acc_stderr": 0.02736586113151381, + "acc_norm": 0.26838235294117646, + "acc_norm_stderr": 0.026917481224377225 + }, + "hendrycksTest-econometrics": { + "acc": 0.23684210526315788, + "acc_stderr": 0.03999423879281336, + "acc_norm": 0.24561403508771928, + "acc_norm_stderr": 0.04049339297748142 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.31, + "acc_norm_stderr": 0.046482319871173156 + }, + "hendrycksTest-astronomy": { + "acc": 0.19078947368421054, + "acc_stderr": 0.031975658210325, + "acc_norm": 0.3092105263157895, + "acc_norm_stderr": 0.037610708698674805 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.3330518018018016, + "likelihood_difference_stderr": 0.409193985708339, + "pct_stereotype": 0.6756756756756757, + "pct_stereotype_stderr": 0.04463366615377136 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.884950657894737, + "likelihood_difference_stderr": 0.253860666777557, + "pct_stereotype": 0.6263157894736842, + "pct_stereotype_stderr": 0.03518990966860906 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.27167630057803466, + "acc_stderr": 0.023948512905468355, + "acc_norm": 0.31213872832369943, + "acc_norm_stderr": 0.02494679222527231 + }, + "hendrycksTest-anatomy": { + "acc": 0.2222222222222222, + "acc_stderr": 0.035914440841969694, + "acc_norm": 0.2222222222222222, + "acc_norm_stderr": 0.035914440841969694 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.25, + "acc_stderr": 0.03039153369274154, + "acc_norm": 0.28921568627450983, + "acc_norm_stderr": 0.03182231867647553 + }, + "winogrande": { + "acc": 0.5098658247829518, + "acc_stderr": 0.014049749833367596 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.26, + "acc_stderr": 0.04408440022768079, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542127 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.24, + "acc_stderr": 0.04292346959909284, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "crows_pairs_french": { + "likelihood_difference": 5.457471489266547, + "likelihood_difference_stderr": 0.13638509811011088, + "pct_stereotype": 0.469886702444842, + "pct_stereotype_stderr": 0.012191128795435453 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.21, + "acc_stderr": 0.04093601807403326, + "acc_norm": 0.34, + "acc_norm_stderr": 0.04760952285695235 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.398557692307692, + "likelihood_difference_stderr": 0.6132408773646376, + "pct_stereotype": 0.6307692307692307, + "pct_stereotype_stderr": 0.06032456592830046 + }, + "hendrycksTest-virology": { + "acc": 0.24096385542168675, + "acc_stderr": 0.0332939411907353, + "acc_norm": 0.25903614457831325, + "acc_norm_stderr": 0.03410646614071856 + }, + "hendrycksTest-formal_logic": { + "acc": 0.29365079365079366, + "acc_stderr": 0.04073524322147127, + "acc_norm": 0.2698412698412698, + "acc_norm_stderr": 0.03970158273235173 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "crows_pairs_english": { + "likelihood_difference": 3.5479185301132974, + "likelihood_difference_stderr": 0.10342276747333694, + "pct_stereotype": 0.5199761478831246, + "pct_stereotype_stderr": 0.012203547977669909 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.17407407407407408, + "acc_stderr": 0.023118596033551844, + "acc_norm": 0.22962962962962963, + "acc_norm_stderr": 0.025644108639267613 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.24352331606217617, + "acc_stderr": 0.03097543638684544, + "acc_norm": 0.2538860103626943, + "acc_norm_stderr": 0.03141024780565319 + }, + "piqa": { + "acc": 0.6022850924918389, + "acc_stderr": 0.011419114133117227, + "acc_norm": 0.5854189336235038, + "acc_norm_stderr": 0.011494326682255153 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.21818181818181817, + "acc_stderr": 0.03225078108306289, + "acc_norm": 0.2787878787878788, + "acc_norm_stderr": 0.03501438706296781 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.461174242424242, + "likelihood_difference_stderr": 0.7416625315218861, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.06176056549879611 + }, + "hendrycksTest-sociology": { + "acc": 0.2885572139303483, + "acc_stderr": 0.03203841040213321, + "acc_norm": 0.25870646766169153, + "acc_norm_stderr": 0.030965903123573037 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.26666666666666666, + "acc_stderr": 0.022421273612923714, + "acc_norm": 0.26153846153846155, + "acc_norm_stderr": 0.02228214120420442 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2515964240102171, + "acc_stderr": 0.015517322365529631, + "acc_norm": 0.26053639846743293, + "acc_norm_stderr": 0.01569600856380708 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.22758620689655173, + "acc_stderr": 0.03493950380131184, + "acc_norm": 0.2896551724137931, + "acc_norm_stderr": 0.03780019230438014 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.26380368098159507, + "acc_stderr": 0.03462419931615624, + "acc_norm": 0.2883435582822086, + "acc_norm_stderr": 0.035590395316173425 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.23178807947019867, + "acc_stderr": 0.03445406271987053, + "acc_norm": 0.2185430463576159, + "acc_norm_stderr": 0.03374235550425694 + }, + "arc_challenge": { + "acc": 0.18344709897610922, + "acc_stderr": 0.011310170179554543, + "acc_norm": 0.2226962457337884, + "acc_norm_stderr": 0.012158314774829928 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.26143790849673204, + "acc_stderr": 0.017776947157528023, + "acc_norm": 0.25, + "acc_norm_stderr": 0.01751781884501444 + }, + "hendrycksTest-management": { + "acc": 0.23300970873786409, + "acc_stderr": 0.04185832598928313, + "acc_norm": 0.2815533980582524, + "acc_norm_stderr": 0.04453254836326468 + }, + "hendrycksTest-machine_learning": { + "acc": 0.2767857142857143, + "acc_stderr": 0.042466243366976256, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04109974682633932 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.2361111111111111, + "acc_stderr": 0.02896370257079103, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.02988691054762698 + }, + "hendrycksTest-college_biology": { + "acc": 0.2361111111111111, + "acc_stderr": 0.03551446610810826, + "acc_norm": 0.2638888888888889, + "acc_norm_stderr": 0.03685651095897532 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.01171875, + "likelihood_difference_stderr": 0.2666371712870024, + "pct_stereotype": 0.515625, + "pct_stereotype_stderr": 0.027980952958187033 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.23628691983122363, + "acc_stderr": 0.027652153144159263, + "acc_norm": 0.29535864978902954, + "acc_norm_stderr": 0.029696338713422896 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.23148148148148148, + "acc_stderr": 0.04077494709252626, + "acc_norm": 0.3611111111111111, + "acc_norm_stderr": 0.04643454608906274 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.4823757381889764, + "likelihood_difference_stderr": 0.1834613447384125, + "pct_stereotype": 0.42913385826771655, + "pct_stereotype_stderr": 0.02198161280908021 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.902667984189724, + "likelihood_difference_stderr": 0.4317932808940784, + "pct_stereotype": 0.2885375494071146, + "pct_stereotype_stderr": 0.028541506394353756 + }, + "hendrycksTest-computer_security": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.26, + "acc_norm_stderr": 0.044084400227680794 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.23529411764705882, + "acc_stderr": 0.027553614467863807, + "acc_norm": 0.33613445378151263, + "acc_norm_stderr": 0.03068473711513537 + }, + "hendrycksTest-prehistory": { + "acc": 0.29012345679012347, + "acc_stderr": 0.025251173936495033, + "acc_norm": 0.24074074074074073, + "acc_norm_stderr": 0.02378858355165854 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.23015873015873015, + "acc_stderr": 0.021679219663693135, + "acc_norm": 0.23544973544973544, + "acc_norm_stderr": 0.02185150982203172 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.6139018691588785, + "likelihood_difference_stderr": 0.23820029270692883, + "pct_stereotype": 0.5327102803738317, + "pct_stereotype_stderr": 0.027890972865217984 + }, + "hendrycksTest-college_physics": { + "acc": 0.19607843137254902, + "acc_stderr": 0.03950581861179963, + "acc_norm": 0.24509803921568626, + "acc_norm_stderr": 0.04280105837364395 + }, + "hendrycksTest-philosophy": { + "acc": 0.20257234726688103, + "acc_stderr": 0.022827317491059682, + "acc_norm": 0.2861736334405145, + "acc_norm_stderr": 0.025670259242188947 + }, + "hendrycksTest-security_studies": { + "acc": 0.3346938775510204, + "acc_stderr": 0.030209235226242307, + "acc_norm": 0.3020408163265306, + "acc_norm_stderr": 0.029393609319879818 + }, + "logiqa": { + "acc": 0.19969278033794163, + "acc_stderr": 0.015680245966420602, + "acc_norm": 0.27956989247311825, + "acc_norm_stderr": 0.017602909186822453 + }, + "hendrycksTest-professional_law": { + "acc": 0.2405475880052151, + "acc_stderr": 0.010916406735478949, + "acc_norm": 0.2777053455019557, + "acc_norm_stderr": 0.011438741422769575 + }, + "hendrycksTest-business_ethics": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206845 + }, + "sciq": { + "acc": 0.628, + "acc_stderr": 0.015292149942040577, + "acc_norm": 0.565, + "acc_norm_stderr": 0.0156850572527172 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.24581005586592178, + "acc_stderr": 0.01440029642922562, + "acc_norm": 0.2346368715083799, + "acc_norm_stderr": 0.014173044098303656 + }, + "hendrycksTest-nutrition": { + "acc": 0.27124183006535946, + "acc_stderr": 0.025457756696667874, + "acc_norm": 0.35294117647058826, + "acc_norm_stderr": 0.02736359328468493 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.24468085106382978, + "acc_stderr": 0.02564555362226673, + "acc_norm": 0.24822695035460993, + "acc_norm_stderr": 0.025770015644290385 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.21132075471698114, + "acc_stderr": 0.02512576648482784, + "acc_norm": 0.3132075471698113, + "acc_norm_stderr": 0.02854479331905533 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 5.027224170918367, + "likelihood_difference_stderr": 0.40815165419858696, + "pct_stereotype": 0.5459183673469388, + "pct_stereotype_stderr": 0.035654431417332814 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 4.238636363636363, + "likelihood_difference_stderr": 1.5362064537668891, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.26382978723404255, + "acc_stderr": 0.028809989854102967, + "acc_norm": 0.19148936170212766, + "acc_norm_stderr": 0.025722149992637805 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.2777777777777778, + "acc_stderr": 0.031911782267135466, + "acc_norm": 0.2828282828282828, + "acc_norm_stderr": 0.0320877955878675 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.24770642201834864, + "acc_stderr": 0.018508143602547825, + "acc_norm": 0.27889908256880735, + "acc_norm_stderr": 0.019227468876463514 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.35, + "acc_stderr": 0.0479372485441102, + "acc_norm": 0.34, + "acc_norm_stderr": 0.04760952285695236 + }, + "hendrycksTest-marketing": { + "acc": 0.3034188034188034, + "acc_stderr": 0.030118210106942666, + "acc_norm": 0.3247863247863248, + "acc_norm_stderr": 0.03067902276549883 + }, + "hendrycksTest-international_law": { + "acc": 0.12396694214876033, + "acc_stderr": 0.030083098716035216, + "acc_norm": 0.38016528925619836, + "acc_norm_stderr": 0.04431324501968431 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.6055975274725274, + "likelihood_difference_stderr": 0.26100601132095375, + "pct_stereotype": 0.5054945054945055, + "pct_stereotype_stderr": 0.052701445311128796 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.138194444444444, + "likelihood_difference_stderr": 0.476805260992122, + "pct_stereotype": 0.4777777777777778, + "pct_stereotype_stderr": 0.05294752255076824 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 6.822458791208791, + "likelihood_difference_stderr": 0.521713971413461, + "pct_stereotype": 0.7912087912087912, + "pct_stereotype_stderr": 0.04284305206509431 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 5.533967391304348, + "likelihood_difference_stderr": 0.5162540247064128, + "pct_stereotype": 0.5739130434782609, + "pct_stereotype_stderr": 0.04631479453711978 + }, + "arc_easy": { + "acc": 0.3707912457912458, + "acc_stderr": 0.009911292822056923, + "acc_norm": 0.3514309764309764, + "acc_norm_stderr": 0.009796395582817719 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.18226600985221675, + "acc_stderr": 0.027163340859645148, + "acc_norm": 0.2561576354679803, + "acc_norm_stderr": 0.030712730070982592 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.25483870967741934, + "acc_stderr": 0.02479011845933221, + "acc_norm": 0.3258064516129032, + "acc_norm_stderr": 0.0266620105785671 + }, + "hendrycksTest-world_religions": { + "acc": 0.2573099415204678, + "acc_stderr": 0.03352799844161865, + "acc_norm": 0.27485380116959063, + "acc_norm_stderr": 0.03424042924691583 + }, + "hendrycksTest-public_relations": { + "acc": 0.3090909090909091, + "acc_stderr": 0.044262946482000985, + "acc_norm": 0.19090909090909092, + "acc_norm_stderr": 0.03764425585984924 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 5.0048076923076925, + "likelihood_difference_stderr": 1.3534970275729887, + "pct_stereotype": 0.38461538461538464, + "pct_stereotype_stderr": 0.1404416814115811 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.743923611111111, + "likelihood_difference_stderr": 0.594433189081222, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.05933908290969268 + }, + "hendrycksTest-global_facts": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.26, + "acc_norm_stderr": 0.04408440022768079 + } + }, + "versions": { + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_french_race_color": 0, + "crows_pairs_english_nationality": 0, + "crows_pairs_english_physical_appearance": 0, + "lambada_openai": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-econometrics": 0, + "wsc": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-astronomy": 0, + "crows_pairs_english_religion": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-high_school_us_history": 0, + "winogrande": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-us_foreign_policy": 0, + "crows_pairs_french": 0, + "hendrycksTest-medical_genetics": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-abstract_algebra": 0, + "crows_pairs_english": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "piqa": 0, + "hendrycksTest-high_school_european_history": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-high_school_physics": 0, + "arc_challenge": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-management": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-college_biology": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-jurisprudence": 0, + "crows_pairs_english_race_color": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-elementary_mathematics": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-security_studies": 0, + "logiqa": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-business_ethics": 0, + "sciq": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_french_socioeconomic": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-international_law": 0, + "crows_pairs_english_age": 0, + "crows_pairs_french_age": 0, + "crows_pairs_french_sexual_orientation": 0, + "crows_pairs_french_religion": 0, + "arc_easy": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-public_relations": 0, + "crows_pairs_french_autre": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-global_facts": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step103000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:2", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step113000.json b/data/checkpoint_eval/eleutherai_evals/step113000.json new file mode 100644 index 0000000000000000000000000000000000000000..52feb9493336bb7610fe4e9111abe43e879f1f95 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step113000.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_english_religion": { + "likelihood_difference": 3.5508164414414414, + "likelihood_difference_stderr": 0.4305168323042274, + "pct_stereotype": 0.6756756756756757, + "pct_stereotype_stderr": 0.04463366615377136 + }, + "hendrycksTest-security_studies": { + "acc": 0.3306122448979592, + "acc_stderr": 0.030116426296540603, + "acc_norm": 0.2938775510204082, + "acc_norm_stderr": 0.029162738410249783 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.26666666666666666, + "acc_stderr": 0.03453131801885417, + "acc_norm": 0.296969696969697, + "acc_norm_stderr": 0.035679697722680495 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.4817913385826773, + "likelihood_difference_stderr": 0.1735254203842936, + "pct_stereotype": 0.44881889763779526, + "pct_stereotype_stderr": 0.022089136921635943 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.086778846153846, + "likelihood_difference_stderr": 0.573459690809164, + "pct_stereotype": 0.6615384615384615, + "pct_stereotype_stderr": 0.059148294227806535 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.22486772486772486, + "acc_stderr": 0.021502096078229147, + "acc_norm": 0.23809523809523808, + "acc_norm_stderr": 0.02193587808118476 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.7198350694444446, + "likelihood_difference_stderr": 0.41714897518097466, + "pct_stereotype": 0.5694444444444444, + "pct_stereotype_stderr": 0.05876396677084613 + }, + "crows_pairs_french": { + "likelihood_difference": 5.481817419499105, + "likelihood_difference_stderr": 0.1368512914780332, + "pct_stereotype": 0.42277877161598093, + "pct_stereotype_stderr": 0.012066761431141063 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.22268907563025211, + "acc_stderr": 0.027025433498882402, + "acc_norm": 0.3277310924369748, + "acc_norm_stderr": 0.030489911417673227 + }, + "arc_easy": { + "acc": 0.3720538720538721, + "acc_stderr": 0.009918187193096463, + "acc_norm": 0.3442760942760943, + "acc_norm_stderr": 0.009749495321590813 + }, + "piqa": { + "acc": 0.5914036996735582, + "acc_stderr": 0.011469240387245139, + "acc_norm": 0.5930359085963003, + "acc_norm_stderr": 0.011462093919190166 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.7811677631578946, + "likelihood_difference_stderr": 0.2573616886550986, + "pct_stereotype": 0.631578947368421, + "pct_stereotype_stderr": 0.03508771929824559 + }, + "hendrycksTest-college_medicine": { + "acc": 0.2658959537572254, + "acc_stderr": 0.033687629322594295, + "acc_norm": 0.34104046242774566, + "acc_norm_stderr": 0.03614665424180826 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.2745664739884393, + "acc_stderr": 0.024027745155265002, + "acc_norm": 0.30057803468208094, + "acc_norm_stderr": 0.024685316867257796 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2554278416347382, + "acc_stderr": 0.015594955384455772, + "acc_norm": 0.25798212005108556, + "acc_norm_stderr": 0.01564583018834895 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 7.007554945054945, + "likelihood_difference_stderr": 0.4540502244754289, + "pct_stereotype": 0.8241758241758241, + "pct_stereotype_stderr": 0.04012619468902319 + }, + "hendrycksTest-sociology": { + "acc": 0.25870646766169153, + "acc_stderr": 0.03096590312357305, + "acc_norm": 0.2935323383084577, + "acc_norm_stderr": 0.03220024104534205 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2748091603053435, + "acc_stderr": 0.03915345408847836, + "acc_norm": 0.2366412213740458, + "acc_norm_stderr": 0.03727673575596919 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 4.708152173913043, + "likelihood_difference_stderr": 0.5092079597931493, + "pct_stereotype": 0.5565217391304348, + "pct_stereotype_stderr": 0.04652911680416962 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.629734848484849, + "likelihood_difference_stderr": 0.6913292787839433, + "pct_stereotype": 0.48484848484848486, + "pct_stereotype_stderr": 0.06198888629778894 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 8.110054347826088, + "likelihood_difference_stderr": 0.4446449895681656, + "pct_stereotype": 0.30434782608695654, + "pct_stereotype_stderr": 0.028985507246376756 + }, + "hendrycksTest-econometrics": { + "acc": 0.2894736842105263, + "acc_stderr": 0.042663394431593935, + "acc_norm": 0.2543859649122807, + "acc_norm_stderr": 0.04096985139843671 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.26838235294117646, + "acc_stderr": 0.026917481224377232, + "acc_norm": 0.23897058823529413, + "acc_norm_stderr": 0.02590528064489301 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.308333333333334, + "likelihood_difference_stderr": 0.48705073491708323, + "pct_stereotype": 0.4666666666666667, + "pct_stereotype_stderr": 0.05288198530254015 + }, + "hendrycksTest-business_ethics": { + "acc": 0.35, + "acc_stderr": 0.047937248544110196, + "acc_norm": 0.34, + "acc_norm_stderr": 0.04760952285695235 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.2222222222222222, + "acc_stderr": 0.028353212866863434, + "acc_norm": 0.25462962962962965, + "acc_norm_stderr": 0.029711275860005344 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.23, + "acc_stderr": 0.04229525846816508, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206824 + }, + "hendrycksTest-prehistory": { + "acc": 0.27469135802469136, + "acc_stderr": 0.024836057868294677, + "acc_norm": 0.2006172839506173, + "acc_norm_stderr": 0.022282313949774875 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.648866758241758, + "likelihood_difference_stderr": 0.28308193114727737, + "pct_stereotype": 0.4725274725274725, + "pct_stereotype_stderr": 0.05262501097748859 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.25738396624472576, + "acc_stderr": 0.028458820991460295, + "acc_norm": 0.3080168776371308, + "acc_norm_stderr": 0.03005238933560569 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.2169976635514015, + "likelihood_difference_stderr": 0.23040467831858316, + "pct_stereotype": 0.4953271028037383, + "pct_stereotype_stderr": 0.02794962902436015 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.27419354838709675, + "acc_stderr": 0.025378139970885203, + "acc_norm": 0.29354838709677417, + "acc_norm_stderr": 0.02590608702131929 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.17592592592592593, + "acc_stderr": 0.03680918141673881, + "acc_norm": 0.3333333333333333, + "acc_norm_stderr": 0.04557239513497752 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.24019607843137256, + "acc_stderr": 0.02998373305591361, + "acc_norm": 0.27941176470588236, + "acc_norm_stderr": 0.031493281045079556 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.002840909090909, + "likelihood_difference_stderr": 1.251678238076205, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.26, + "acc_stderr": 0.044084400227680794, + "acc_norm": 0.21, + "acc_norm_stderr": 0.04093601807403326 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.26422018348623855, + "acc_stderr": 0.018904164171510193, + "acc_norm": 0.22752293577981653, + "acc_norm_stderr": 0.017974463578776502 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.16, + "acc_stderr": 0.03684529491774709, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909282 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.1625615763546798, + "acc_stderr": 0.025960300064605576, + "acc_norm": 0.2512315270935961, + "acc_norm_stderr": 0.030516530732694436 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.24150943396226415, + "acc_stderr": 0.026341480371118355, + "acc_norm": 0.3471698113207547, + "acc_norm_stderr": 0.029300101705549652 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2375886524822695, + "acc_stderr": 0.025389512552729906, + "acc_norm": 0.2624113475177305, + "acc_norm_stderr": 0.026244920349843 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 5.202686543367347, + "likelihood_difference_stderr": 0.40401351786153467, + "pct_stereotype": 0.44387755102040816, + "pct_stereotype_stderr": 0.035579471949536604 + }, + "sciq": { + "acc": 0.617, + "acc_stderr": 0.015380102325652713, + "acc_norm": 0.557, + "acc_norm_stderr": 0.0157161699532041 + }, + "winogrande": { + "acc": 0.5185477505919495, + "acc_stderr": 0.014042813708888378 + }, + "hendrycksTest-philosophy": { + "acc": 0.22186495176848875, + "acc_stderr": 0.02359885829286305, + "acc_norm": 0.2765273311897106, + "acc_norm_stderr": 0.02540383297817961 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.24, + "acc_stderr": 0.04292346959909282, + "acc_norm": 0.36, + "acc_norm_stderr": 0.04824181513244218 + }, + "hendrycksTest-astronomy": { + "acc": 0.23684210526315788, + "acc_stderr": 0.03459777606810536, + "acc_norm": 0.3223684210526316, + "acc_norm_stderr": 0.038035102483515854 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.24539877300613497, + "acc_stderr": 0.03380939813943354, + "acc_norm": 0.294478527607362, + "acc_norm_stderr": 0.03581165790474082 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "hendrycksTest-anatomy": { + "acc": 0.1925925925925926, + "acc_stderr": 0.03406542058502652, + "acc_norm": 0.22962962962962963, + "acc_norm_stderr": 0.03633384414073462 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.25163398692810457, + "acc_stderr": 0.01755581809132226, + "acc_norm": 0.25326797385620914, + "acc_norm_stderr": 0.01759348689536683 + }, + "hendrycksTest-virology": { + "acc": 0.24096385542168675, + "acc_stderr": 0.0332939411907353, + "acc_norm": 0.25903614457831325, + "acc_norm_stderr": 0.03410646614071856 + }, + "hendrycksTest-professional_law": { + "acc": 0.25684485006518903, + "acc_stderr": 0.011158455853098838, + "acc_norm": 0.28226857887874834, + "acc_norm_stderr": 0.011495852176241935 + }, + "hendrycksTest-international_law": { + "acc": 0.1652892561983471, + "acc_stderr": 0.03390780612972776, + "acc_norm": 0.4049586776859504, + "acc_norm_stderr": 0.044811377559424694 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.657986111111111, + "likelihood_difference_stderr": 0.6715285667233309, + "pct_stereotype": 0.5277777777777778, + "pct_stereotype_stderr": 0.05924743948371487 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "acc_norm": 0.33, + "acc_norm_stderr": 0.04725815626252606 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.23076923076923078, + "acc_stderr": 0.02136202772522271, + "acc_norm": 0.30256410256410254, + "acc_norm_stderr": 0.023290888053772725 + }, + "hendrycksTest-formal_logic": { + "acc": 0.31746031746031744, + "acc_stderr": 0.041634530313028585, + "acc_norm": 0.3253968253968254, + "acc_norm_stderr": 0.041905964388711366 + }, + "hendrycksTest-college_physics": { + "acc": 0.2647058823529412, + "acc_stderr": 0.043898699568087785, + "acc_norm": 0.2549019607843137, + "acc_norm_stderr": 0.04336432707993177 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.28, + "acc_stderr": 0.04512608598542128, + "acc_norm": 0.28, + "acc_norm_stderr": 0.045126085985421276 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.4522569444444446, + "likelihood_difference_stderr": 0.2616537892601734, + "pct_stereotype": 0.4398148148148148, + "pct_stereotype_stderr": 0.03385177976044812 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.2727272727272727, + "acc_stderr": 0.03173071239071724, + "acc_norm": 0.29797979797979796, + "acc_norm_stderr": 0.03258630383836556 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.26382978723404255, + "acc_stderr": 0.02880998985410297, + "acc_norm": 0.2297872340425532, + "acc_norm_stderr": 0.027501752944412417 + }, + "hendrycksTest-marketing": { + "acc": 0.3034188034188034, + "acc_stderr": 0.03011821010694265, + "acc_norm": 0.28205128205128205, + "acc_norm_stderr": 0.02948036054954119 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.32413793103448274, + "acc_stderr": 0.03900432069185555, + "acc_norm": 0.296551724137931, + "acc_norm_stderr": 0.03806142687309993 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.20207253886010362, + "acc_stderr": 0.02897908979429673, + "acc_norm": 0.2538860103626943, + "acc_norm_stderr": 0.03141024780565318 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23687150837988827, + "acc_stderr": 0.014219570788103986, + "acc_norm": 0.264804469273743, + "acc_norm_stderr": 0.01475690648326066 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.14814814814814814, + "acc_stderr": 0.02165977842211803, + "acc_norm": 0.2074074074074074, + "acc_norm_stderr": 0.024720713193952158 + }, + "hendrycksTest-world_religions": { + "acc": 0.2046783625730994, + "acc_stderr": 0.030944459778533207, + "acc_norm": 0.29239766081871343, + "acc_norm_stderr": 0.03488647713457922 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.879872311827957, + "likelihood_difference_stderr": 0.5526395273732472, + "pct_stereotype": 0.8494623655913979, + "pct_stereotype_stderr": 0.03728212869390004 + }, + "hendrycksTest-global_facts": { + "acc": 0.19, + "acc_stderr": 0.03942772444036625, + "acc_norm": 0.21, + "acc_norm_stderr": 0.040936018074033256 + }, + "hendrycksTest-machine_learning": { + "acc": 0.22321428571428573, + "acc_stderr": 0.039523019677025116, + "acc_norm": 0.20535714285714285, + "acc_norm_stderr": 0.038342410214190735 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.058642578125, + "likelihood_difference_stderr": 0.26754943414520593, + "pct_stereotype": 0.54375, + "pct_stereotype_stderr": 0.02788725270865466 + }, + "hendrycksTest-human_aging": { + "acc": 0.2825112107623318, + "acc_stderr": 0.03021683101150876, + "acc_norm": 0.24663677130044842, + "acc_norm_stderr": 0.028930413120910874 + }, + "hendrycksTest-college_biology": { + "acc": 0.2777777777777778, + "acc_stderr": 0.03745554791462458, + "acc_norm": 0.24305555555555555, + "acc_norm_stderr": 0.03586879280080341 + }, + "hendrycksTest-management": { + "acc": 0.18446601941747573, + "acc_stderr": 0.03840423627288276, + "acc_norm": 0.27184466019417475, + "acc_norm_stderr": 0.044052680241409216 + }, + "logiqa": { + "acc": 0.2227342549923195, + "acc_stderr": 0.01632005404616511, + "acc_norm": 0.29493087557603687, + "acc_norm_stderr": 0.01788624973410439 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 5.012771739130435, + "likelihood_difference_stderr": 0.25654636026247285, + "pct_stereotype": 0.28043478260869564, + "pct_stereotype_stderr": 0.02096740469164158 + }, + "hendrycksTest-nutrition": { + "acc": 0.24183006535947713, + "acc_stderr": 0.024518195641879334, + "acc_norm": 0.3333333333333333, + "acc_norm_stderr": 0.026992544339297233 + }, + "arc_challenge": { + "acc": 0.1757679180887372, + "acc_stderr": 0.01112285086312048, + "acc_norm": 0.21928327645051193, + "acc_norm_stderr": 0.012091245787615725 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 3.8533653846153846, + "likelihood_difference_stderr": 1.121278927826028, + "pct_stereotype": 0.46153846153846156, + "pct_stereotype_stderr": 0.14390989949130548 + }, + "hendrycksTest-public_relations": { + "acc": 0.2818181818181818, + "acc_stderr": 0.043091187099464585, + "acc_norm": 0.2, + "acc_norm_stderr": 0.03831305140884603 + }, + "lambada_openai": { + "ppl": 120.54804436871059, + "ppl_stderr": 5.111543992276259, + "acc": 0.21618474674946633, + "acc_stderr": 0.005734973987279193 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.24503311258278146, + "acc_stderr": 0.03511807571804725, + "acc_norm": 0.2251655629139073, + "acc_norm_stderr": 0.03410435282008937 + }, + "hendrycksTest-computer_security": { + "acc": 0.26, + "acc_stderr": 0.0440844002276808, + "acc_norm": 0.27, + "acc_norm_stderr": 0.0446196043338474 + }, + "crows_pairs_english": { + "likelihood_difference": 3.550471451997615, + "likelihood_difference_stderr": 0.10173760358787075, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.012177111585868344 + } + }, + "versions": { + "crows_pairs_english_religion": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-high_school_european_history": 0, + "crows_pairs_english_race_color": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-elementary_mathematics": 0, + "crows_pairs_english_physical_appearance": 0, + "crows_pairs_french": 0, + "hendrycksTest-high_school_microeconomics": 0, + "arc_easy": 0, + "piqa": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-miscellaneous": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_french_religion": 0, + "crows_pairs_french_disability": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-professional_medicine": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-high_school_statistics": 0, + "wsc": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-prehistory": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-high_school_world_history": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-professional_accounting": 0, + "crows_pairs_french_socioeconomic": 0, + "sciq": 0, + "winogrande": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-international_law": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-us_foreign_policy": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-machine_learning": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-management": 0, + "logiqa": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-nutrition": 0, + "arc_challenge": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-public_relations": 0, + "lambada_openai": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-computer_security": 0, + "crows_pairs_english": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step113000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:3", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step123000.json b/data/checkpoint_eval/eleutherai_evals/step123000.json new file mode 100644 index 0000000000000000000000000000000000000000..9a613188e9d717c653ebfed4b8ad502888af3350 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step123000.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_english_age": { + "likelihood_difference": 2.6893887362637363, + "likelihood_difference_stderr": 0.2680170470214308, + "pct_stereotype": 0.5494505494505495, + "pct_stereotype_stderr": 0.05244623100101224 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.2, + "acc_stderr": 0.040201512610368445, + "acc_norm": 0.28, + "acc_norm_stderr": 0.045126085985421276 + }, + "hendrycksTest-college_medicine": { + "acc": 0.26011560693641617, + "acc_stderr": 0.03345036916788991, + "acc_norm": 0.28901734104046245, + "acc_norm_stderr": 0.034564257450869995 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 4.231534090909091, + "likelihood_difference_stderr": 1.4093980202807441, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-college_physics": { + "acc": 0.19607843137254902, + "acc_stderr": 0.03950581861179964, + "acc_norm": 0.22549019607843138, + "acc_norm_stderr": 0.04158307533083286 + }, + "hendrycksTest-prehistory": { + "acc": 0.25617283950617287, + "acc_stderr": 0.0242885336377261, + "acc_norm": 0.2345679012345679, + "acc_norm_stderr": 0.023576881744005716 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816505 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.25326797385620914, + "acc_stderr": 0.01759348689536683, + "acc_norm": 0.2581699346405229, + "acc_norm_stderr": 0.01770453165325007 + }, + "hendrycksTest-management": { + "acc": 0.1941747572815534, + "acc_stderr": 0.03916667762822584, + "acc_norm": 0.32038834951456313, + "acc_norm_stderr": 0.0462028408228004 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 6.262362637362638, + "likelihood_difference_stderr": 0.4610653877296226, + "pct_stereotype": 0.8021978021978022, + "pct_stereotype_stderr": 0.04198895203196222 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.503783218503937, + "likelihood_difference_stderr": 0.1827001921291112, + "pct_stereotype": 0.47834645669291337, + "pct_stereotype_stderr": 0.022184946299954114 + }, + "hendrycksTest-philosophy": { + "acc": 0.21221864951768488, + "acc_stderr": 0.023222756797435122, + "acc_norm": 0.26688102893890675, + "acc_norm_stderr": 0.025122637608816632 + }, + "logiqa": { + "acc": 0.21658986175115208, + "acc_stderr": 0.0161568605831783, + "acc_norm": 0.29185867895545314, + "acc_norm_stderr": 0.01783157055397193 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 4.449519230769231, + "likelihood_difference_stderr": 0.9182419616034272, + "pct_stereotype": 0.3076923076923077, + "pct_stereotype_stderr": 0.13323467750529824 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.5115384615384615, + "likelihood_difference_stderr": 0.5769147322613659, + "pct_stereotype": 0.6461538461538462, + "pct_stereotype_stderr": 0.05977027026123099 + }, + "arc_challenge": { + "acc": 0.17747440273037543, + "acc_stderr": 0.011165138769643972, + "acc_norm": 0.23037542662116042, + "acc_norm_stderr": 0.01230492841874761 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.23636363636363636, + "acc_stderr": 0.033175059300091805, + "acc_norm": 0.24848484848484848, + "acc_norm_stderr": 0.033744026441394036 + }, + "hendrycksTest-international_law": { + "acc": 0.17355371900826447, + "acc_stderr": 0.03457272836917671, + "acc_norm": 0.371900826446281, + "acc_norm_stderr": 0.04412015806624504 + }, + "hendrycksTest-formal_logic": { + "acc": 0.30158730158730157, + "acc_stderr": 0.04104947269903394, + "acc_norm": 0.2857142857142857, + "acc_norm_stderr": 0.0404061017820884 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2656449553001277, + "acc_stderr": 0.015794302487888715, + "acc_norm": 0.24521072796934865, + "acc_norm_stderr": 0.015384352284543944 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.2694300518134715, + "acc_stderr": 0.03201867122877794, + "acc_norm": 0.26424870466321243, + "acc_norm_stderr": 0.03182155050916647 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.18518518518518517, + "acc_stderr": 0.03755265865037181, + "acc_norm": 0.37037037037037035, + "acc_norm_stderr": 0.04668408033024931 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.19907407407407407, + "acc_stderr": 0.02723229846269024, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.02988691054762698 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.26717557251908397, + "acc_stderr": 0.03880848301082395, + "acc_norm": 0.25190839694656486, + "acc_norm_stderr": 0.03807387116306086 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.7981770833333335, + "likelihood_difference_stderr": 0.4326219499395864, + "pct_stereotype": 0.625, + "pct_stereotype_stderr": 0.05745481997211521 + }, + "crows_pairs_english": { + "likelihood_difference": 3.580799045915325, + "likelihood_difference_stderr": 0.10390408640595741, + "pct_stereotype": 0.5468097793679189, + "pct_stereotype_stderr": 0.012159658951661536 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "acc_norm": 0.35, + "acc_norm_stderr": 0.04793724854411021 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.21509433962264152, + "acc_stderr": 0.025288394502891363, + "acc_norm": 0.2981132075471698, + "acc_norm_stderr": 0.02815283794249386 + }, + "arc_easy": { + "acc": 0.36574074074074076, + "acc_stderr": 0.009882988069418829, + "acc_norm": 0.34385521885521886, + "acc_norm_stderr": 0.009746660584852448 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.2658959537572254, + "acc_stderr": 0.023786203255508277, + "acc_norm": 0.30057803468208094, + "acc_norm_stderr": 0.024685316867257796 + }, + "hendrycksTest-college_biology": { + "acc": 0.2708333333333333, + "acc_stderr": 0.03716177437566014, + "acc_norm": 0.2569444444444444, + "acc_norm_stderr": 0.03653946969442099 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.2616033755274262, + "acc_stderr": 0.028609516716994934, + "acc_norm": 0.270042194092827, + "acc_norm_stderr": 0.028900721906293426 + }, + "hendrycksTest-nutrition": { + "acc": 0.26143790849673204, + "acc_stderr": 0.025160998214292456, + "acc_norm": 0.3431372549019608, + "acc_norm_stderr": 0.027184498909941616 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.32, + "acc_norm_stderr": 0.046882617226215034 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.23, + "acc_stderr": 0.04229525846816508, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542127 + }, + "hendrycksTest-econometrics": { + "acc": 0.2543859649122807, + "acc_stderr": 0.04096985139843671, + "acc_norm": 0.24561403508771928, + "acc_norm_stderr": 0.04049339297748141 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.4215277777777775, + "likelihood_difference_stderr": 0.4586681734751123, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.052999894000318 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.1814814814814815, + "acc_stderr": 0.0234992646694073, + "acc_norm": 0.23703703703703705, + "acc_norm_stderr": 0.0259288761327661 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 4.770652173913043, + "likelihood_difference_stderr": 0.5382010210337033, + "pct_stereotype": 0.4782608695652174, + "pct_stereotype_stderr": 0.04678500755208439 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2827586206896552, + "acc_stderr": 0.03752833958003337, + "acc_norm": 0.30344827586206896, + "acc_norm_stderr": 0.038312260488503336 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.583423913043478, + "likelihood_difference_stderr": 0.25993020575528686, + "pct_stereotype": 0.3869565217391304, + "pct_stereotype_stderr": 0.02273371341289454 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 5.456672512755102, + "likelihood_difference_stderr": 0.4238880074973118, + "pct_stereotype": 0.5153061224489796, + "pct_stereotype_stderr": 0.03578896281770489 + }, + "hendrycksTest-security_studies": { + "acc": 0.27755102040816326, + "acc_stderr": 0.02866685779027465, + "acc_norm": 0.24897959183673468, + "acc_norm_stderr": 0.027682979522960234 + }, + "wsc": { + "acc": 0.375, + "acc_stderr": 0.04770204856076104 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.26422018348623855, + "acc_stderr": 0.018904164171510186, + "acc_norm": 0.25137614678899084, + "acc_norm_stderr": 0.018599206360287415 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.443341121495327, + "likelihood_difference_stderr": 0.22479210100562896, + "pct_stereotype": 0.5327102803738317, + "pct_stereotype_stderr": 0.027890972865217977 + }, + "lambada_openai": { + "ppl": 133.11058088169239, + "ppl_stderr": 5.629178590421373, + "acc": 0.20415292062875995, + "acc_stderr": 0.005615710162255026 + }, + "winogrande": { + "acc": 0.5027624309392266, + "acc_stderr": 0.014052271211616436 + }, + "hendrycksTest-marketing": { + "acc": 0.2692307692307692, + "acc_stderr": 0.029058588303748845, + "acc_norm": 0.32051282051282054, + "acc_norm_stderr": 0.030572811310299607 + }, + "hendrycksTest-public_relations": { + "acc": 0.3090909090909091, + "acc_stderr": 0.044262946482000985, + "acc_norm": 0.2, + "acc_norm_stderr": 0.03831305140884601 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.23529411764705882, + "acc_stderr": 0.027553614467863818, + "acc_norm": 0.3403361344537815, + "acc_norm_stderr": 0.030778057422931673 + }, + "hendrycksTest-astronomy": { + "acc": 0.21710526315789475, + "acc_stderr": 0.033550453048829226, + "acc_norm": 0.3815789473684211, + "acc_norm_stderr": 0.03953173377749194 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.526475694444445, + "likelihood_difference_stderr": 0.6334563568346864, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.05933908290969268 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.15270935960591134, + "acc_stderr": 0.02530890453938063, + "acc_norm": 0.26108374384236455, + "acc_norm_stderr": 0.030903796952114468 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.20245398773006135, + "acc_stderr": 0.03157065078911901, + "acc_norm": 0.2883435582822086, + "acc_norm_stderr": 0.035590395316173425 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.24838709677419354, + "acc_stderr": 0.02458002892148101, + "acc_norm": 0.3225806451612903, + "acc_norm_stderr": 0.02659308451657228 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.2119205298013245, + "acc_stderr": 0.03336767086567978, + "acc_norm": 0.2251655629139073, + "acc_norm_stderr": 0.03410435282008936 + }, + "piqa": { + "acc": 0.5859630032644179, + "acc_stderr": 0.011492118481417575, + "acc_norm": 0.5875952121871599, + "acc_norm_stderr": 0.011485407152743137 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.33, + "acc_stderr": 0.04725815626252604, + "acc_norm": 0.32, + "acc_norm_stderr": 0.04688261722621503 + }, + "hendrycksTest-human_aging": { + "acc": 0.2645739910313901, + "acc_stderr": 0.029605103217038332, + "acc_norm": 0.2556053811659193, + "acc_norm_stderr": 0.029275891003969927 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.5471565315315314, + "likelihood_difference_stderr": 0.4012612135638327, + "pct_stereotype": 0.6396396396396397, + "pct_stereotype_stderr": 0.045776211670703136 + }, + "hendrycksTest-professional_law": { + "acc": 0.23663624511082137, + "acc_stderr": 0.010855137351572737, + "acc_norm": 0.26140808344198174, + "acc_norm_stderr": 0.011222528169771309 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.406754032258065, + "likelihood_difference_stderr": 0.5393566741888578, + "pct_stereotype": 0.7741935483870968, + "pct_stereotype_stderr": 0.043591220947882314 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.047314453125, + "likelihood_difference_stderr": 0.28201771347734694, + "pct_stereotype": 0.546875, + "pct_stereotype_stderr": 0.027871330781745147 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.70998023715415, + "likelihood_difference_stderr": 0.4337314450386741, + "pct_stereotype": 0.3201581027667984, + "pct_stereotype_stderr": 0.029389076633931355 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.21957671957671956, + "acc_stderr": 0.021320018599770355, + "acc_norm": 0.2222222222222222, + "acc_norm_stderr": 0.021411684393694196 + }, + "sciq": { + "acc": 0.614, + "acc_stderr": 0.01540263747678437, + "acc_norm": 0.559, + "acc_norm_stderr": 0.015708779894242676 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.24822695035460993, + "acc_stderr": 0.025770015644290382, + "acc_norm": 0.25886524822695034, + "acc_norm_stderr": 0.026129572527180848 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.25757575757575757, + "acc_stderr": 0.031156269519646847, + "acc_norm": 0.31313131313131315, + "acc_norm_stderr": 0.03304205087813653 + }, + "hendrycksTest-sociology": { + "acc": 0.2835820895522388, + "acc_stderr": 0.03187187537919796, + "acc_norm": 0.24875621890547264, + "acc_norm_stderr": 0.030567675938916714 + }, + "hendrycksTest-virology": { + "acc": 0.26506024096385544, + "acc_stderr": 0.03436024037944967, + "acc_norm": 0.25301204819277107, + "acc_norm_stderr": 0.03384429155233137 + }, + "hendrycksTest-business_ethics": { + "acc": 0.33, + "acc_stderr": 0.04725815626252604, + "acc_norm": 0.34, + "acc_norm_stderr": 0.04760952285695235 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.24134078212290502, + "acc_stderr": 0.014310999547961443, + "acc_norm": 0.24916201117318434, + "acc_norm_stderr": 0.01446589382985993 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.24509803921568626, + "acc_stderr": 0.03019028245350194, + "acc_norm": 0.2696078431372549, + "acc_norm_stderr": 0.031145570659486782 + }, + "hendrycksTest-machine_learning": { + "acc": 0.24107142857142858, + "acc_stderr": 0.040598672469526864, + "acc_norm": 0.23214285714285715, + "acc_norm_stderr": 0.04007341809755807 + }, + "hendrycksTest-global_facts": { + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "acc_norm": 0.24, + "acc_norm_stderr": 0.042923469599092816 + }, + "hendrycksTest-computer_security": { + "acc": 0.22, + "acc_stderr": 0.041633319989322695, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206845 + }, + "crows_pairs_french": { + "likelihood_difference": 5.343344700357782, + "likelihood_difference_stderr": 0.1361582396406217, + "pct_stereotype": 0.46153846153846156, + "pct_stereotype_stderr": 0.012177111585868348 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.572443181818182, + "likelihood_difference_stderr": 0.7053414331507932, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.06176056549879611 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.519458912037037, + "likelihood_difference_stderr": 0.26480319253010903, + "pct_stereotype": 0.4305555555555556, + "pct_stereotype_stderr": 0.03376922151252336 + }, + "hendrycksTest-world_religions": { + "acc": 0.23391812865497075, + "acc_stderr": 0.03246721765117825, + "acc_norm": 0.26900584795321636, + "acc_norm_stderr": 0.03401052620104089 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 4.016694078947369, + "likelihood_difference_stderr": 0.25038749794237203, + "pct_stereotype": 0.6368421052631579, + "pct_stereotype_stderr": 0.03498104083833203 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.23161764705882354, + "acc_stderr": 0.025626533803777562, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.026799562024887674 + }, + "hendrycksTest-anatomy": { + "acc": 0.2740740740740741, + "acc_stderr": 0.03853254836552003, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.037857144650666544 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.24871794871794872, + "acc_stderr": 0.0219169577092138, + "acc_norm": 0.26666666666666666, + "acc_norm_stderr": 0.022421273612923714 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.34, + "acc_norm_stderr": 0.04760952285695235 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2723404255319149, + "acc_stderr": 0.0291012906983867, + "acc_norm": 0.20425531914893616, + "acc_norm_stderr": 0.02635515841334942 + } + }, + "versions": { + "crows_pairs_english_age": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-management": 0, + "crows_pairs_french_sexual_orientation": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-philosophy": 0, + "logiqa": 0, + "crows_pairs_french_autre": 0, + "crows_pairs_english_disability": 0, + "arc_challenge": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_english_physical_appearance": 0, + "crows_pairs_english": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-clinical_knowledge": 0, + "arc_easy": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-econometrics": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-high_school_mathematics": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-electrical_engineering": 0, + "crows_pairs_french_race_color": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-security_studies": 0, + "wsc": 0, + "hendrycksTest-high_school_psychology": 0, + "crows_pairs_french_gender": 0, + "lambada_openai": 0, + "winogrande": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-astronomy": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-high_school_physics": 0, + "piqa": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-human_aging": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-professional_law": 0, + "crows_pairs_english_sexual_orientation": 0, + "crows_pairs_english_gender": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-elementary_mathematics": 0, + "sciq": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-computer_security": 0, + "crows_pairs_french": 0, + "crows_pairs_french_disability": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-conceptual_physics": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step123000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:4", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step128.json b/data/checkpoint_eval/eleutherai_evals/step128.json new file mode 100644 index 0000000000000000000000000000000000000000..9963f4c151b6c177cd0c29d6f11b954a29af1a99 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step128.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 8.304253472222221, + "likelihood_difference_stderr": 1.1645432805257396, + "pct_stereotype": 0.5694444444444444, + "pct_stereotype_stderr": 0.05876396677084613 + }, + "logiqa": { + "acc": 0.20276497695852536, + "acc_stderr": 0.015770046635584567, + "acc_norm": 0.22734254992319508, + "acc_norm_stderr": 0.016439067675117765 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.25287356321839083, + "acc_stderr": 0.015543377313719681, + "acc_norm": 0.26309067688378035, + "acc_norm_stderr": 0.015745497169049043 + }, + "hendrycksTest-machine_learning": { + "acc": 0.1875, + "acc_stderr": 0.0370468111477387, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04109974682633932 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 8.420701581027668, + "likelihood_difference_stderr": 0.40347811300443653, + "pct_stereotype": 0.6047430830039525, + "pct_stereotype_stderr": 0.030798170848773863 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.2222222222222222, + "acc_stderr": 0.02835321286686343, + "acc_norm": 0.2638888888888889, + "acc_norm_stderr": 0.03005820270430985 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.19, + "acc_stderr": 0.03942772444036624, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.22727272727272727, + "acc_stderr": 0.029857515673386417, + "acc_norm": 0.2474747474747475, + "acc_norm_stderr": 0.03074630074212451 + }, + "hendrycksTest-college_biology": { + "acc": 0.20833333333333334, + "acc_stderr": 0.03396116205845335, + "acc_norm": 0.24305555555555555, + "acc_norm_stderr": 0.03586879280080341 + }, + "hendrycksTest-philosophy": { + "acc": 0.27009646302250806, + "acc_stderr": 0.025218040373410616, + "acc_norm": 0.27009646302250806, + "acc_norm_stderr": 0.02521804037341062 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.20634920634920634, + "acc_stderr": 0.02084229093011467, + "acc_norm": 0.2222222222222222, + "acc_norm_stderr": 0.021411684393694185 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 7.021634615384615, + "likelihood_difference_stderr": 0.97111821919133, + "pct_stereotype": 0.47692307692307695, + "pct_stereotype_stderr": 0.062433396464415106 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.677083333333333, + "likelihood_difference_stderr": 0.6378864031151475, + "pct_stereotype": 0.5053763440860215, + "pct_stereotype_stderr": 0.05212558986469174 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 10.822225765306122, + "likelihood_difference_stderr": 0.6868426421668223, + "pct_stereotype": 0.3010204081632653, + "pct_stereotype_stderr": 0.0328483010552734 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "hendrycksTest-security_studies": { + "acc": 0.3183673469387755, + "acc_stderr": 0.02982253379398205, + "acc_norm": 0.22857142857142856, + "acc_norm_stderr": 0.02688214492230775 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 5.092623873873874, + "likelihood_difference_stderr": 0.5870374592750236, + "pct_stereotype": 0.5045045045045045, + "pct_stereotype_stderr": 0.047671194793956616 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.32061068702290074, + "acc_stderr": 0.04093329229834278, + "acc_norm": 0.31297709923664124, + "acc_norm_stderr": 0.04066962905677697 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.673295454545454, + "likelihood_difference_stderr": 2.5462810016897417, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-nutrition": { + "acc": 0.1895424836601307, + "acc_stderr": 0.022442358263336216, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.02526169121972948 + }, + "wsc": { + "acc": 0.6346153846153846, + "acc_stderr": 0.0474473339327792 + }, + "crows_pairs_french": { + "likelihood_difference": 8.28959824090638, + "likelihood_difference_stderr": 0.1813110538245477, + "pct_stereotype": 0.5724508050089445, + "pct_stereotype_stderr": 0.012084400901134948 + }, + "hendrycksTest-marketing": { + "acc": 0.20512820512820512, + "acc_stderr": 0.02645350805404033, + "acc_norm": 0.23076923076923078, + "acc_norm_stderr": 0.027601921381417607 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.20725388601036268, + "acc_stderr": 0.02925282329180363, + "acc_norm": 0.27461139896373055, + "acc_norm_stderr": 0.03221024508041156 + }, + "hendrycksTest-professional_law": { + "acc": 0.23533246414602346, + "acc_stderr": 0.010834432543912219, + "acc_norm": 0.2653194263363755, + "acc_norm_stderr": 0.011276198843958866 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.2, + "acc_stderr": 0.04020151261036846, + "acc_norm": 0.23, + "acc_norm_stderr": 0.042295258468165044 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542129 + }, + "hendrycksTest-college_medicine": { + "acc": 0.2254335260115607, + "acc_stderr": 0.03186209851641144, + "acc_norm": 0.27167630057803466, + "acc_norm_stderr": 0.03391750322321659 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.2119205298013245, + "acc_stderr": 0.03336767086567977, + "acc_norm": 0.26490066225165565, + "acc_norm_stderr": 0.03603038545360384 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.19032258064516128, + "acc_stderr": 0.022331707611823078, + "acc_norm": 0.2161290322580645, + "acc_norm_stderr": 0.023415293433568515 + }, + "lambada_openai": { + "ppl": 1665636.981895382, + "ppl_stderr": 131507.1881823213, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 13.809065934065934, + "likelihood_difference_stderr": 0.8840201832791948, + "pct_stereotype": 0.8021978021978022, + "pct_stereotype_stderr": 0.04198895203196222 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "acc_norm": 0.15, + "acc_norm_stderr": 0.03588702812826372 + }, + "hendrycksTest-business_ethics": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "acc_norm": 0.35, + "acc_norm_stderr": 0.04793724854411019 + }, + "hendrycksTest-human_aging": { + "acc": 0.2825112107623318, + "acc_stderr": 0.03021683101150877, + "acc_norm": 0.27802690582959644, + "acc_norm_stderr": 0.030069584874494033 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.21098265895953758, + "acc_stderr": 0.021966309947043117, + "acc_norm": 0.2023121387283237, + "acc_norm_stderr": 0.021628077380196137 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 9.863315217391305, + "likelihood_difference_stderr": 0.6835325659384403, + "pct_stereotype": 0.6782608695652174, + "pct_stereotype_stderr": 0.04375199868936841 + }, + "hendrycksTest-sociology": { + "acc": 0.2736318407960199, + "acc_stderr": 0.031524391865554044, + "acc_norm": 0.26865671641791045, + "acc_norm_stderr": 0.03134328358208954 + }, + "piqa": { + "acc": 0.5261153427638737, + "acc_stderr": 0.011649900854263423, + "acc_norm": 0.5184983677910773, + "acc_norm_stderr": 0.01165783758381816 + }, + "arc_challenge": { + "acc": 0.20136518771331058, + "acc_stderr": 0.011718927477444262, + "acc_norm": 0.23890784982935154, + "acc_norm_stderr": 0.012461071376316617 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.18067226890756302, + "acc_stderr": 0.02499196496660076, + "acc_norm": 0.27310924369747897, + "acc_norm_stderr": 0.028942004040998167 + }, + "arc_easy": { + "acc": 0.2676767676767677, + "acc_stderr": 0.009085000147099353, + "acc_norm": 0.2756734006734007, + "acc_norm_stderr": 0.009169229476542562 + }, + "winogrande": { + "acc": 0.49171270718232046, + "acc_stderr": 0.014050555322824192 + }, + "hendrycksTest-astronomy": { + "acc": 0.18421052631578946, + "acc_stderr": 0.031546980450822305, + "acc_norm": 0.29605263157894735, + "acc_norm_stderr": 0.03715062154998905 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.24468085106382978, + "acc_stderr": 0.02564555362226673, + "acc_norm": 0.25886524822695034, + "acc_norm_stderr": 0.026129572527180848 + }, + "sciq": { + "acc": 0.219, + "acc_stderr": 0.013084731950262024, + "acc_norm": 0.225, + "acc_norm_stderr": 0.013211720158614751 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.23, + "acc_stderr": 0.042295258468165044, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206845 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.3021276595744681, + "acc_stderr": 0.030017554471880557, + "acc_norm": 0.2723404255319149, + "acc_norm_stderr": 0.029101290698386698 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.15757575757575756, + "acc_stderr": 0.02845038880528435, + "acc_norm": 0.22424242424242424, + "acc_norm_stderr": 0.03256866661681102 + }, + "hendrycksTest-management": { + "acc": 0.23300970873786409, + "acc_stderr": 0.04185832598928315, + "acc_norm": 0.2815533980582524, + "acc_norm_stderr": 0.04453254836326466 + }, + "crows_pairs_english": { + "likelihood_difference": 5.005841905187835, + "likelihood_difference_stderr": 0.1594255961602287, + "pct_stereotype": 0.4442456768038163, + "pct_stereotype_stderr": 0.012137130534698495 + }, + "hendrycksTest-public_relations": { + "acc": 0.2727272727272727, + "acc_stderr": 0.04265792110940588, + "acc_norm": 0.16363636363636364, + "acc_norm_stderr": 0.03543433054298678 + }, + "hendrycksTest-computer_security": { + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206845 + }, + "hendrycksTest-virology": { + "acc": 0.18674698795180722, + "acc_stderr": 0.030338749144500618, + "acc_norm": 0.23493975903614459, + "acc_norm_stderr": 0.03300533186128922 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.14814814814814814, + "acc_stderr": 0.02165977842211803, + "acc_norm": 0.22592592592592592, + "acc_norm_stderr": 0.02549753263960955 + }, + "hendrycksTest-anatomy": { + "acc": 0.18518518518518517, + "acc_stderr": 0.03355677216313142, + "acc_norm": 0.26666666666666666, + "acc_norm_stderr": 0.038201699145179055 + }, + "hendrycksTest-formal_logic": { + "acc": 0.2857142857142857, + "acc_stderr": 0.04040610178208841, + "acc_norm": 0.2777777777777778, + "acc_norm_stderr": 0.040061680838488774 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.1574074074074074, + "acc_stderr": 0.035207039905179656, + "acc_norm": 0.23148148148148148, + "acc_norm_stderr": 0.04077494709252627 + }, + "hendrycksTest-international_law": { + "acc": 0.09090909090909091, + "acc_stderr": 0.026243194054073878, + "acc_norm": 0.2727272727272727, + "acc_norm_stderr": 0.04065578140908705 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 6.225961538461538, + "likelihood_difference_stderr": 1.0845778061093316, + "pct_stereotype": 0.6153846153846154, + "pct_stereotype_stderr": 0.14044168141158106 + }, + "hendrycksTest-global_facts": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816506 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 4.482080078125, + "likelihood_difference_stderr": 0.4507987456254625, + "pct_stereotype": 0.54375, + "pct_stereotype_stderr": 0.027887252708654657 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 6.005063657407407, + "likelihood_difference_stderr": 0.4386043275416813, + "pct_stereotype": 0.3194444444444444, + "pct_stereotype_stderr": 0.0317987634217685 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 4.779035433070866, + "likelihood_difference_stderr": 0.26206678827654106, + "pct_stereotype": 0.3346456692913386, + "pct_stereotype_stderr": 0.02095632470166831 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 7.161277173913043, + "likelihood_difference_stderr": 0.26238666011376527, + "pct_stereotype": 0.7108695652173913, + "pct_stereotype_stderr": 0.02116096760624947 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 12.47064393939394, + "likelihood_difference_stderr": 1.234102889806993, + "pct_stereotype": 0.3333333333333333, + "pct_stereotype_stderr": 0.0584705346204686 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.19831223628691982, + "acc_stderr": 0.025955020841621115, + "acc_norm": 0.2616033755274262, + "acc_norm_stderr": 0.028609516716994934 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.20245398773006135, + "acc_stderr": 0.031570650789119, + "acc_norm": 0.3006134969325153, + "acc_norm_stderr": 0.03602511318806771 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.18719211822660098, + "acc_stderr": 0.027444924966882618, + "acc_norm": 0.21674876847290642, + "acc_norm_stderr": 0.02899033125251624 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.21, + "acc_norm_stderr": 0.040936018074033256 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.382638888888889, + "likelihood_difference_stderr": 0.563720955887399, + "pct_stereotype": 0.4666666666666667, + "pct_stereotype_stderr": 0.05288198530254015 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.21025641025641026, + "acc_stderr": 0.020660597485026938, + "acc_norm": 0.26153846153846155, + "acc_norm_stderr": 0.022282141204204426 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.24183006535947713, + "acc_stderr": 0.017322789207784326, + "acc_norm": 0.26633986928104575, + "acc_norm_stderr": 0.017883188134667164 + }, + "hendrycksTest-econometrics": { + "acc": 0.21929824561403508, + "acc_stderr": 0.03892431106518752, + "acc_norm": 0.2719298245614035, + "acc_norm_stderr": 0.04185774424022057 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.299793956043956, + "likelihood_difference_stderr": 0.48606624275634924, + "pct_stereotype": 0.4835164835164835, + "pct_stereotype_stderr": 0.05267597952306975 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.18490566037735848, + "acc_stderr": 0.02389335183446432, + "acc_norm": 0.30943396226415093, + "acc_norm_stderr": 0.028450154794118627 + }, + "hendrycksTest-college_physics": { + "acc": 0.17647058823529413, + "acc_stderr": 0.0379328118530781, + "acc_norm": 0.17647058823529413, + "acc_norm_stderr": 0.037932811853078105 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 5.136284722222222, + "likelihood_difference_stderr": 0.6655017702667131, + "pct_stereotype": 0.5277777777777778, + "pct_stereotype_stderr": 0.05924743948371487 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 6.4443146417445485, + "likelihood_difference_stderr": 0.3439159663666914, + "pct_stereotype": 0.48909657320872274, + "pct_stereotype_stderr": 0.02794420307081864 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.19117647058823528, + "acc_stderr": 0.02759917430064077, + "acc_norm": 0.23039215686274508, + "acc_norm_stderr": 0.029554292605695063 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135302, + "acc_norm": 0.2620689655172414, + "acc_norm_stderr": 0.036646663372252565 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 5.01858552631579, + "likelihood_difference_stderr": 0.4299366436044498, + "pct_stereotype": 0.5842105263157895, + "pct_stereotype_stderr": 0.0358501132552001 + }, + "hendrycksTest-prehistory": { + "acc": 0.2623456790123457, + "acc_stderr": 0.024477222856135118, + "acc_norm": 0.24691358024691357, + "acc_norm_stderr": 0.023993501709042124 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.23669724770642203, + "acc_stderr": 0.01822407811729907, + "acc_norm": 0.24770642201834864, + "acc_norm_stderr": 0.018508143602547815 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.26838235294117646, + "acc_stderr": 0.026917481224377232, + "acc_norm": 0.28308823529411764, + "acc_norm_stderr": 0.027365861131513805 + }, + "hendrycksTest-world_religions": { + "acc": 0.1871345029239766, + "acc_stderr": 0.029913127232368043, + "acc_norm": 0.23976608187134502, + "acc_norm_stderr": 0.032744852119469564 + } + }, + "versions": { + "crows_pairs_french_physical_appearance": 0, + "logiqa": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-machine_learning": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-elementary_mathematics": 0, + "crows_pairs_english_disability": 0, + "crows_pairs_english_sexual_orientation": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-security_studies": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-nutrition": 0, + "wsc": 0, + "crows_pairs_french": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-high_school_biology": 0, + "lambada_openai": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-moral_disputes": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-sociology": 0, + "piqa": 0, + "arc_challenge": 0, + "hendrycksTest-high_school_microeconomics": 0, + "arc_easy": 0, + "winogrande": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-professional_accounting": 0, + "sciq": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-management": 0, + "crows_pairs_english": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-international_law": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-global_facts": 0, + "crows_pairs_english_gender": 0, + "crows_pairs_english_nationality": 0, + "crows_pairs_english_race_color": 0, + "crows_pairs_french_race_color": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-college_computer_science": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-econometrics": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-college_physics": 0, + "crows_pairs_english_physical_appearance": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-electrical_engineering": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-world_religions": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step128", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step13000.json b/data/checkpoint_eval/eleutherai_evals/step13000.json new file mode 100644 index 0000000000000000000000000000000000000000..fd5102be619a8d3f85102fc457bc2ff6c5334839 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step13000.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-high_school_world_history": { + "acc": 0.25738396624472576, + "acc_stderr": 0.028458820991460267, + "acc_norm": 0.3037974683544304, + "acc_norm_stderr": 0.029936696387138615 + }, + "lambada_openai": { + "ppl": 136.24789967804702, + "ppl_stderr": 5.841149004257701, + "acc": 0.21036289540073744, + "acc_stderr": 0.005678196483274596 + }, + "winogrande": { + "acc": 0.5027624309392266, + "acc_stderr": 0.014052271211616441 + }, + "hendrycksTest-world_religions": { + "acc": 0.28654970760233917, + "acc_stderr": 0.03467826685703826, + "acc_norm": 0.34502923976608185, + "acc_norm_stderr": 0.036459813773888065 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.7294560185185186, + "likelihood_difference_stderr": 0.28249531206763423, + "pct_stereotype": 0.39351851851851855, + "pct_stereotype_stderr": 0.03331747876370312 + }, + "hendrycksTest-professional_law": { + "acc": 0.24771838331160365, + "acc_stderr": 0.011025499291443738, + "acc_norm": 0.2770534550195567, + "acc_norm_stderr": 0.01143046244371968 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.25210084033613445, + "acc_stderr": 0.02820554503327771, + "acc_norm": 0.3277310924369748, + "acc_norm_stderr": 0.030489911417673227 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.24, + "acc_stderr": 0.04292346959909282, + "acc_norm": 0.37, + "acc_norm_stderr": 0.04852365870939099 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.2331288343558282, + "acc_stderr": 0.0332201579577674, + "acc_norm": 0.2822085889570552, + "acc_norm_stderr": 0.03536117886664743 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 5.96875, + "likelihood_difference_stderr": 0.6136503830214869, + "pct_stereotype": 0.8021978021978022, + "pct_stereotype_stderr": 0.04198895203196222 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.24871794871794872, + "acc_stderr": 0.021916957709213803, + "acc_norm": 0.26153846153846155, + "acc_norm_stderr": 0.022282141204204426 + }, + "hendrycksTest-college_biology": { + "acc": 0.2361111111111111, + "acc_stderr": 0.03551446610810826, + "acc_norm": 0.2222222222222222, + "acc_norm_stderr": 0.03476590104304134 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.23529411764705882, + "acc_stderr": 0.029771775228145628, + "acc_norm": 0.24509803921568626, + "acc_norm_stderr": 0.03019028245350195 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 4.261439732142857, + "likelihood_difference_stderr": 0.3621777755494698, + "pct_stereotype": 0.42857142857142855, + "pct_stereotype_stderr": 0.035438495596916704 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.22, + "acc_stderr": 0.041633319989322695, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909282 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.111616847826087, + "likelihood_difference_stderr": 0.21388555403339707, + "pct_stereotype": 0.4391304347826087, + "pct_stereotype_stderr": 0.023164416405982075 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2689655172413793, + "acc_stderr": 0.036951833116502325, + "acc_norm": 0.31724137931034485, + "acc_norm_stderr": 0.03878352372138623 + }, + "hendrycksTest-sociology": { + "acc": 0.2935323383084577, + "acc_stderr": 0.03220024104534205, + "acc_norm": 0.2835820895522388, + "acc_norm_stderr": 0.03187187537919797 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.16666666666666666, + "acc_stderr": 0.022722578464550523, + "acc_norm": 0.25555555555555554, + "acc_norm_stderr": 0.026593939101844075 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.26055045871559634, + "acc_stderr": 0.01881918203485007, + "acc_norm": 0.24220183486238533, + "acc_norm_stderr": 0.01836817630659862 + }, + "piqa": { + "acc": 0.5865070729053319, + "acc_stderr": 0.011489895831821131, + "acc_norm": 0.5930359085963003, + "acc_norm_stderr": 0.011462093919190166 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 5.045380434782609, + "likelihood_difference_stderr": 0.5425900004201588, + "pct_stereotype": 0.40869565217391307, + "pct_stereotype_stderr": 0.04604188749503788 + }, + "hendrycksTest-machine_learning": { + "acc": 0.26785714285714285, + "acc_stderr": 0.04203277291467764, + "acc_norm": 0.19642857142857142, + "acc_norm_stderr": 0.03770970049347018 + }, + "hendrycksTest-econometrics": { + "acc": 0.2894736842105263, + "acc_stderr": 0.04266339443159394, + "acc_norm": 0.2543859649122807, + "acc_norm_stderr": 0.04096985139843671 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.21, + "acc_norm_stderr": 0.040936018074033256 + }, + "logiqa": { + "acc": 0.2012288786482335, + "acc_stderr": 0.01572532582742823, + "acc_norm": 0.2780337941628264, + "acc_norm_stderr": 0.017573187770282713 + }, + "hendrycksTest-formal_logic": { + "acc": 0.3253968253968254, + "acc_stderr": 0.041905964388711366, + "acc_norm": 0.29365079365079366, + "acc_norm_stderr": 0.040735243221471255 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.25132275132275134, + "acc_stderr": 0.022340482339643895, + "acc_norm": 0.2777777777777778, + "acc_norm_stderr": 0.02306818884826111 + }, + "hendrycksTest-international_law": { + "acc": 0.12396694214876033, + "acc_stderr": 0.030083098716035237, + "acc_norm": 0.4049586776859504, + "acc_norm_stderr": 0.044811377559424694 + }, + "hendrycksTest-virology": { + "acc": 0.21686746987951808, + "acc_stderr": 0.03208284450356365, + "acc_norm": 0.2469879518072289, + "acc_norm_stderr": 0.03357351982064536 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.7626644736842105, + "likelihood_difference_stderr": 0.263472659726499, + "pct_stereotype": 0.6421052631578947, + "pct_stereotype_stderr": 0.03486983309720002 + }, + "hendrycksTest-computer_security": { + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542127 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.159722222222222, + "likelihood_difference_stderr": 0.6604052452240913, + "pct_stereotype": 0.4861111111111111, + "pct_stereotype_stderr": 0.05931618532716555 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.23829787234042554, + "acc_stderr": 0.027851252973889764, + "acc_norm": 0.19574468085106383, + "acc_norm_stderr": 0.025937853139977145 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.35, + "acc_stderr": 0.04793724854411019, + "acc_norm": 0.35, + "acc_norm_stderr": 0.0479372485441102 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.22264150943396227, + "acc_stderr": 0.025604233470899098, + "acc_norm": 0.3169811320754717, + "acc_norm_stderr": 0.028637235639800928 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.21212121212121213, + "acc_stderr": 0.029126522834586832, + "acc_norm": 0.26262626262626265, + "acc_norm_stderr": 0.031353050095330855 + }, + "sciq": { + "acc": 0.65, + "acc_stderr": 0.015090650341444231, + "acc_norm": 0.577, + "acc_norm_stderr": 0.01563058909047635 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 3.9399038461538463, + "likelihood_difference_stderr": 0.93832403380636, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.14390989949130545 + }, + "hendrycksTest-prehistory": { + "acc": 0.25925925925925924, + "acc_stderr": 0.02438366553103545, + "acc_norm": 0.18518518518518517, + "acc_norm_stderr": 0.0216138093952248 + }, + "crows_pairs_french": { + "likelihood_difference": 5.003405448717949, + "likelihood_difference_stderr": 0.12449132819328972, + "pct_stereotype": 0.4364937388193202, + "pct_stereotype_stderr": 0.012114385095725013 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.6161417322834644, + "likelihood_difference_stderr": 0.18602106212344902, + "pct_stereotype": 0.421259842519685, + "pct_stereotype_stderr": 0.021928698676414303 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.23121387283236994, + "acc_stderr": 0.022698657167855713, + "acc_norm": 0.31213872832369943, + "acc_norm_stderr": 0.02494679222527231 + }, + "hendrycksTest-college_medicine": { + "acc": 0.21965317919075145, + "acc_stderr": 0.031568093627031744, + "acc_norm": 0.3468208092485549, + "acc_norm_stderr": 0.036291466701596636 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.2610294117647059, + "acc_stderr": 0.026679252270103128, + "acc_norm": 0.27205882352941174, + "acc_norm_stderr": 0.027033041151681456 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.296875, + "likelihood_difference_stderr": 2.3872581014414274, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-security_studies": { + "acc": 0.3551020408163265, + "acc_stderr": 0.030635655150387634, + "acc_norm": 0.24489795918367346, + "acc_norm_stderr": 0.02752963744017493 + }, + "crows_pairs_french_age": { + "likelihood_difference": 3.9774305555555554, + "likelihood_difference_stderr": 0.38401151759615354, + "pct_stereotype": 0.4222222222222222, + "pct_stereotype_stderr": 0.05235473399540657 + }, + "hendrycksTest-human_aging": { + "acc": 0.2600896860986547, + "acc_stderr": 0.029442495585857476, + "acc_norm": 0.22869955156950672, + "acc_norm_stderr": 0.02818824004692919 + }, + "hendrycksTest-management": { + "acc": 0.20388349514563106, + "acc_stderr": 0.039891398595317706, + "acc_norm": 0.2524271844660194, + "acc_norm_stderr": 0.04301250399690878 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.199961059190031, + "likelihood_difference_stderr": 0.19666947220858882, + "pct_stereotype": 0.5077881619937694, + "pct_stereotype_stderr": 0.02794745876935634 + }, + "hendrycksTest-global_facts": { + "acc": 0.2, + "acc_stderr": 0.04020151261036847, + "acc_norm": 0.2, + "acc_norm_stderr": 0.04020151261036846 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.824004120879121, + "likelihood_difference_stderr": 0.25515072273014766, + "pct_stereotype": 0.5714285714285714, + "pct_stereotype_stderr": 0.05216405309573015 + }, + "hendrycksTest-business_ethics": { + "acc": 0.32, + "acc_stderr": 0.046882617226215034, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542128 + }, + "hendrycksTest-anatomy": { + "acc": 0.23703703703703705, + "acc_stderr": 0.03673731683969506, + "acc_norm": 0.21481481481481482, + "acc_norm_stderr": 0.03547854198560824 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.17592592592592593, + "acc_stderr": 0.036809181416738807, + "acc_norm": 0.3425925925925926, + "acc_norm_stderr": 0.04587904741301812 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.25177304964539005, + "acc_stderr": 0.0258921511567094, + "acc_norm": 0.2624113475177305, + "acc_norm_stderr": 0.026244920349843007 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.1724137931034483, + "acc_stderr": 0.0265776721830366, + "acc_norm": 0.2660098522167488, + "acc_norm_stderr": 0.03108982600293752 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.19, + "acc_stderr": 0.03942772444036622, + "acc_norm": 0.29, + "acc_norm_stderr": 0.04560480215720683 + }, + "hendrycksTest-college_physics": { + "acc": 0.17647058823529413, + "acc_stderr": 0.0379328118530781, + "acc_norm": 0.24509803921568626, + "acc_norm_stderr": 0.042801058373643966 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.19689119170984457, + "acc_stderr": 0.028697873971860677, + "acc_norm": 0.29533678756476683, + "acc_norm_stderr": 0.03292296639155141 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.058837890625, + "likelihood_difference_stderr": 0.2502387813084938, + "pct_stereotype": 0.56875, + "pct_stereotype_stderr": 0.027728726065513788 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.24692737430167597, + "acc_norm_stderr": 0.014422292204808836 + }, + "hendrycksTest-public_relations": { + "acc": 0.23636363636363636, + "acc_stderr": 0.040693063197213754, + "acc_norm": 0.16363636363636364, + "acc_norm_stderr": 0.035434330542986774 + }, + "crows_pairs_english": { + "likelihood_difference": 3.6419387298747763, + "likelihood_difference_stderr": 0.10530121451159095, + "pct_stereotype": 0.5336911150864639, + "pct_stereotype_stderr": 0.012185541257180466 + }, + "arc_challenge": { + "acc": 0.17491467576791808, + "acc_stderr": 0.011101562501828234, + "acc_norm": 0.22184300341296928, + "acc_norm_stderr": 0.012141659068147887 + }, + "hendrycksTest-marketing": { + "acc": 0.27350427350427353, + "acc_stderr": 0.029202540153431197, + "acc_norm": 0.28205128205128205, + "acc_norm_stderr": 0.02948036054954119 + }, + "hendrycksTest-philosophy": { + "acc": 0.2057877813504823, + "acc_stderr": 0.022961339906764248, + "acc_norm": 0.2829581993569132, + "acc_norm_stderr": 0.025583062489984824 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2503192848020434, + "acc_stderr": 0.015491088951494576, + "acc_norm": 0.24393358876117496, + "acc_norm_stderr": 0.015357212665829475 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.3603603603603602, + "likelihood_difference_stderr": 0.39596313038117587, + "pct_stereotype": 0.6666666666666666, + "pct_stereotype_stderr": 0.04494665749754944 + }, + "hendrycksTest-nutrition": { + "acc": 0.22549019607843138, + "acc_stderr": 0.023929155517351284, + "acc_norm": 0.33986928104575165, + "acc_norm_stderr": 0.027121956071388852 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.2119205298013245, + "acc_stderr": 0.03336767086567977, + "acc_norm": 0.2185430463576159, + "acc_norm_stderr": 0.033742355504256936 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.25483870967741934, + "acc_stderr": 0.024790118459332208, + "acc_norm": 0.32903225806451614, + "acc_norm_stderr": 0.02672949906834997 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.554567307692308, + "likelihood_difference_stderr": 0.6793924919190663, + "pct_stereotype": 0.6461538461538462, + "pct_stereotype_stderr": 0.05977027026123099 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.31297709923664124, + "acc_stderr": 0.04066962905677698, + "acc_norm": 0.3053435114503817, + "acc_norm_stderr": 0.04039314978724561 + }, + "arc_easy": { + "acc": 0.3952020202020202, + "acc_stderr": 0.01003189405279098, + "acc_norm": 0.3522727272727273, + "acc_norm_stderr": 0.009801753933112771 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.968413978494624, + "likelihood_difference_stderr": 0.6082206813914695, + "pct_stereotype": 0.8064516129032258, + "pct_stereotype_stderr": 0.041189832133487855 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.775938735177865, + "likelihood_difference_stderr": 0.3805565031745419, + "pct_stereotype": 0.233201581027668, + "pct_stereotype_stderr": 0.026638273845497516 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.24019607843137256, + "acc_stderr": 0.017282760695167418, + "acc_norm": 0.25980392156862747, + "acc_norm_stderr": 0.017740899509177795 + }, + "hendrycksTest-astronomy": { + "acc": 0.20394736842105263, + "acc_stderr": 0.0327900040631005, + "acc_norm": 0.34210526315789475, + "acc_norm_stderr": 0.03860731599316092 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.735795454545454, + "likelihood_difference_stderr": 0.7110577603651055, + "pct_stereotype": 0.36363636363636365, + "pct_stereotype_stderr": 0.05966637484671758 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.23636363636363636, + "acc_stderr": 0.033175059300091805, + "acc_norm": 0.3090909090909091, + "acc_norm_stderr": 0.036085410115739666 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.14351851851851852, + "acc_stderr": 0.02391077925264438, + "acc_norm": 0.2222222222222222, + "acc_norm_stderr": 0.028353212866863434 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.609375, + "likelihood_difference_stderr": 0.41664133487403315, + "pct_stereotype": 0.6111111111111112, + "pct_stereotype_stderr": 0.057855371034784615 + } + }, + "versions": { + "hendrycksTest-high_school_world_history": 0, + "lambada_openai": 0, + "winogrande": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-logical_fallacies": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-college_mathematics": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-high_school_mathematics": 0, + "wsc": 0, + "hendrycksTest-high_school_psychology": 0, + "piqa": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-college_computer_science": 0, + "logiqa": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-virology": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-computer_security": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-high_school_geography": 0, + "sciq": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-prehistory": 0, + "crows_pairs_french": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-professional_medicine": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-security_studies": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-management": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-global_facts": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-public_relations": 0, + "crows_pairs_english": 0, + "arc_challenge": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-miscellaneous": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-high_school_biology": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-human_sexuality": 0, + "arc_easy": 0, + "crows_pairs_english_sexual_orientation": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-abstract_algebra": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-high_school_statistics": 0, + "crows_pairs_english_physical_appearance": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step13000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:1", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step133000.json b/data/checkpoint_eval/eleutherai_evals/step133000.json new file mode 100644 index 0000000000000000000000000000000000000000..57628fb48a56da8f16ae7a8020b3bf6cf707894b --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step133000.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_english_gender": { + "likelihood_difference": 2.968603515625, + "likelihood_difference_stderr": 0.27632562920815934, + "pct_stereotype": 0.55625, + "pct_stereotype_stderr": 0.02781690795790493 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.35, + "acc_stderr": 0.047937248544110196, + "acc_norm": 0.35, + "acc_norm_stderr": 0.047937248544110196 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.533975856697819, + "likelihood_difference_stderr": 0.23505554828445724, + "pct_stereotype": 0.5295950155763239, + "pct_stereotype_stderr": 0.027901844420051187 + }, + "hendrycksTest-machine_learning": { + "acc": 0.26785714285714285, + "acc_stderr": 0.04203277291467763, + "acc_norm": 0.26785714285714285, + "acc_norm_stderr": 0.04203277291467762 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.25957446808510637, + "acc_stderr": 0.02865917937429232, + "acc_norm": 0.20425531914893616, + "acc_norm_stderr": 0.02635515841334942 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.651982060185185, + "likelihood_difference_stderr": 0.2683321501875231, + "pct_stereotype": 0.4074074074074074, + "pct_stereotype_stderr": 0.03350991604696042 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.28, + "acc_stderr": 0.04512608598542126, + "acc_norm": 0.34, + "acc_norm_stderr": 0.047609522856952365 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.26, + "acc_stderr": 0.04408440022768079, + "acc_norm": 0.26, + "acc_norm_stderr": 0.04408440022768079 + }, + "logiqa": { + "acc": 0.20890937019969277, + "acc_stderr": 0.015945399396423927, + "acc_norm": 0.2764976958525346, + "acc_norm_stderr": 0.01754320907582518 + }, + "hendrycksTest-human_aging": { + "acc": 0.2600896860986547, + "acc_stderr": 0.029442495585857483, + "acc_norm": 0.21524663677130046, + "acc_norm_stderr": 0.02758406660220826 + }, + "hendrycksTest-virology": { + "acc": 0.28313253012048195, + "acc_stderr": 0.03507295431370519, + "acc_norm": 0.25903614457831325, + "acc_norm_stderr": 0.03410646614071856 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.518028846153846, + "likelihood_difference_stderr": 0.6386232385225414, + "pct_stereotype": 0.6615384615384615, + "pct_stereotype_stderr": 0.059148294227806535 + }, + "hendrycksTest-security_studies": { + "acc": 0.34285714285714286, + "acc_stderr": 0.030387262919547724, + "acc_norm": 0.2530612244897959, + "acc_norm_stderr": 0.02783302387139968 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.539185531496063, + "likelihood_difference_stderr": 0.16924052532261086, + "pct_stereotype": 0.5118110236220472, + "pct_stereotype_stderr": 0.022199583294816916 + }, + "hendrycksTest-college_medicine": { + "acc": 0.2543352601156069, + "acc_stderr": 0.0332055644308557, + "acc_norm": 0.28901734104046245, + "acc_norm_stderr": 0.034564257450870016 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.23834196891191708, + "acc_stderr": 0.030748905363909902, + "acc_norm": 0.2849740932642487, + "acc_norm_stderr": 0.03257714077709661 + }, + "crows_pairs_french": { + "likelihood_difference": 5.358885845259392, + "likelihood_difference_stderr": 0.13730895956322042, + "pct_stereotype": 0.43828264758497315, + "pct_stereotype_stderr": 0.012119900409052399 + }, + "hendrycksTest-formal_logic": { + "acc": 0.29365079365079366, + "acc_stderr": 0.040735243221471276, + "acc_norm": 0.2857142857142857, + "acc_norm_stderr": 0.040406101782088394 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.2331288343558282, + "acc_stderr": 0.0332201579577674, + "acc_norm": 0.3128834355828221, + "acc_norm_stderr": 0.03642914578292405 + }, + "hendrycksTest-world_religions": { + "acc": 0.21052631578947367, + "acc_stderr": 0.031267817146631786, + "acc_norm": 0.26900584795321636, + "acc_norm_stderr": 0.0340105262010409 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.822203947368421, + "likelihood_difference_stderr": 0.25490868312334425, + "pct_stereotype": 0.6157894736842106, + "pct_stereotype_stderr": 0.03538097998767891 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.270042194092827, + "acc_stderr": 0.028900721906293426, + "acc_norm": 0.25738396624472576, + "acc_norm_stderr": 0.028458820991460285 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.25921787709497207, + "acc_stderr": 0.014655780837497731, + "acc_norm": 0.24692737430167597, + "acc_norm_stderr": 0.014422292204808836 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.2037037037037037, + "acc_stderr": 0.02455617221914125, + "acc_norm": 0.26296296296296295, + "acc_norm_stderr": 0.026842057873833706 + }, + "crows_pairs_english": { + "likelihood_difference": 3.589249776386404, + "likelihood_difference_stderr": 0.10257069027145715, + "pct_stereotype": 0.5497912939773405, + "pct_stereotype_stderr": 0.012152590574174898 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 4.9423076923076925, + "likelihood_difference_stderr": 1.2264143363162354, + "pct_stereotype": 0.46153846153846156, + "pct_stereotype_stderr": 0.14390989949130548 + }, + "hendrycksTest-business_ethics": { + "acc": 0.33, + "acc_stderr": 0.047258156262526045, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.2426470588235294, + "acc_stderr": 0.026040662474201275, + "acc_norm": 0.26838235294117646, + "acc_norm_stderr": 0.02691748122437722 + }, + "piqa": { + "acc": 0.5984766050054406, + "acc_stderr": 0.011437324373397846, + "acc_norm": 0.5865070729053319, + "acc_norm_stderr": 0.011489895831821136 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.26011560693641617, + "acc_stderr": 0.023618678310069363, + "acc_norm": 0.3092485549132948, + "acc_norm_stderr": 0.024883140570071755 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.23404255319148937, + "acc_stderr": 0.025257861359432428, + "acc_norm": 0.25886524822695034, + "acc_norm_stderr": 0.026129572527180848 + }, + "winogrande": { + "acc": 0.5232833464877664, + "acc_stderr": 0.014037241309573642 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.29411764705882354, + "acc_stderr": 0.02959732973097809, + "acc_norm": 0.3697478991596639, + "acc_norm_stderr": 0.031357095996135904 + }, + "hendrycksTest-prehistory": { + "acc": 0.32098765432098764, + "acc_stderr": 0.025976566010862737, + "acc_norm": 0.22530864197530864, + "acc_norm_stderr": 0.02324620264781975 + }, + "arc_easy": { + "acc": 0.3707912457912458, + "acc_stderr": 0.009911292822056923, + "acc_norm": 0.34553872053872053, + "acc_norm_stderr": 0.009757948730670301 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.25660377358490566, + "acc_stderr": 0.026880647889051968, + "acc_norm": 0.3320754716981132, + "acc_norm_stderr": 0.02898545565233439 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.2222222222222222, + "acc_stderr": 0.02835321286686343, + "acc_norm": 0.2361111111111111, + "acc_norm_stderr": 0.028963702570791044 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 5.0625, + "likelihood_difference_stderr": 0.4286061255414444, + "pct_stereotype": 0.7362637362637363, + "pct_stereotype_stderr": 0.04644942852497396 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.2, + "acc_stderr": 0.04020151261036846, + "acc_norm": 0.36, + "acc_norm_stderr": 0.04824181513244218 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.24074074074074073, + "acc_stderr": 0.0220190800122179, + "acc_norm": 0.25132275132275134, + "acc_norm_stderr": 0.022340482339643895 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.770776098901099, + "likelihood_difference_stderr": 0.2827474692485144, + "pct_stereotype": 0.5494505494505495, + "pct_stereotype_stderr": 0.05244623100101224 + }, + "hendrycksTest-international_law": { + "acc": 0.18181818181818182, + "acc_stderr": 0.03520893951097655, + "acc_norm": 0.4380165289256198, + "acc_norm_stderr": 0.045291468044357915 + }, + "lambada_openai": { + "ppl": 148.4586759416483, + "ppl_stderr": 6.263736488398032, + "acc": 0.19347952648942363, + "acc_stderr": 0.005503478560447365 + }, + "arc_challenge": { + "acc": 0.17491467576791808, + "acc_stderr": 0.01110156250182823, + "acc_norm": 0.22013651877133106, + "acc_norm_stderr": 0.012108124883460988 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.508333333333334, + "likelihood_difference_stderr": 0.4073546275703716, + "pct_stereotype": 0.36666666666666664, + "pct_stereotype_stderr": 0.05108070528032164 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.2696078431372549, + "acc_stderr": 0.031145570659486782, + "acc_norm": 0.27450980392156865, + "acc_norm_stderr": 0.031321798030832904 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.19, + "acc_stderr": 0.039427724440366234, + "acc_norm": 0.26, + "acc_norm_stderr": 0.04408440022768078 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.30344827586206896, + "acc_stderr": 0.038312260488503336, + "acc_norm": 0.2827586206896552, + "acc_norm_stderr": 0.037528339580033376 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 4.394886363636363, + "likelihood_difference_stderr": 1.3966545911585055, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-professional_law": { + "acc": 0.24315514993481094, + "acc_stderr": 0.010956556654417346, + "acc_norm": 0.273142112125163, + "acc_norm_stderr": 0.011380150567830396 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.2037037037037037, + "acc_stderr": 0.03893542518824847, + "acc_norm": 0.3425925925925926, + "acc_norm_stderr": 0.04587904741301811 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.31297709923664124, + "acc_stderr": 0.04066962905677698, + "acc_norm": 0.2366412213740458, + "acc_norm_stderr": 0.037276735755969195 + }, + "hendrycksTest-public_relations": { + "acc": 0.2545454545454545, + "acc_stderr": 0.04172343038705383, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.03694284335337798 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.710665760869565, + "likelihood_difference_stderr": 0.2548812285925958, + "pct_stereotype": 0.35434782608695653, + "pct_stereotype_stderr": 0.02232584228256916 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.2806451612903226, + "acc_stderr": 0.02556060472102288, + "acc_norm": 0.3096774193548387, + "acc_norm_stderr": 0.026302774983517418 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.26422018348623855, + "acc_stderr": 0.0189041641715102, + "acc_norm": 0.25688073394495414, + "acc_norm_stderr": 0.018732492928342465 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.2545454545454545, + "acc_stderr": 0.03401506715249039, + "acc_norm": 0.32727272727272727, + "acc_norm_stderr": 0.03663974994391242 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.26262626262626265, + "acc_stderr": 0.031353050095330855, + "acc_norm": 0.29292929292929293, + "acc_norm_stderr": 0.03242497958178815 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.517809139784946, + "likelihood_difference_stderr": 0.5509527681471865, + "pct_stereotype": 0.7096774193548387, + "pct_stereotype_stderr": 0.04732351421824122 + }, + "hendrycksTest-philosophy": { + "acc": 0.21221864951768488, + "acc_stderr": 0.02322275679743512, + "acc_norm": 0.24758842443729903, + "acc_norm_stderr": 0.024513879973621967 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.21182266009852216, + "acc_stderr": 0.02874898368994106, + "acc_norm": 0.28078817733990147, + "acc_norm_stderr": 0.03161856335358609 + }, + "hendrycksTest-college_biology": { + "acc": 0.25, + "acc_stderr": 0.03621034121889507, + "acc_norm": 0.2708333333333333, + "acc_norm_stderr": 0.03716177437566017 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 5.1596380739795915, + "likelihood_difference_stderr": 0.43468629725261915, + "pct_stereotype": 0.46938775510204084, + "pct_stereotype_stderr": 0.035738572888608724 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.549268018018018, + "likelihood_difference_stderr": 0.4167156533862711, + "pct_stereotype": 0.6396396396396397, + "pct_stereotype_stderr": 0.04577621167070314 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.535037878787879, + "likelihood_difference_stderr": 0.7494925060241215, + "pct_stereotype": 0.4696969696969697, + "pct_stereotype_stderr": 0.06190336468479955 + }, + "hendrycksTest-marketing": { + "acc": 0.2905982905982906, + "acc_stderr": 0.029745048572674054, + "acc_norm": 0.3034188034188034, + "acc_norm_stderr": 0.030118210106942645 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.26666666666666666, + "acc_stderr": 0.022421273612923714, + "acc_norm": 0.30256410256410254, + "acc_norm_stderr": 0.02329088805377273 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.2434640522875817, + "acc_stderr": 0.017362473762146637, + "acc_norm": 0.24509803921568626, + "acc_norm_stderr": 0.01740181671142766 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.9303385416666665, + "likelihood_difference_stderr": 0.4204840146607544, + "pct_stereotype": 0.6111111111111112, + "pct_stereotype_stderr": 0.057855371034784615 + }, + "hendrycksTest-econometrics": { + "acc": 0.2631578947368421, + "acc_stderr": 0.041424397194893624, + "acc_norm": 0.2543859649122807, + "acc_norm_stderr": 0.040969851398436716 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 5.23695652173913, + "likelihood_difference_stderr": 0.5548918938463742, + "pct_stereotype": 0.5391304347826087, + "pct_stereotype_stderr": 0.04668566114758418 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.778532608695652, + "likelihood_difference_stderr": 0.44524534158910173, + "pct_stereotype": 0.2924901185770751, + "pct_stereotype_stderr": 0.028656396908494267 + }, + "hendrycksTest-anatomy": { + "acc": 0.2222222222222222, + "acc_stderr": 0.035914440841969694, + "acc_norm": 0.22962962962962963, + "acc_norm_stderr": 0.03633384414073463 + }, + "sciq": { + "acc": 0.617, + "acc_stderr": 0.01538010232565271, + "acc_norm": 0.541, + "acc_norm_stderr": 0.015766025737882165 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.18543046357615894, + "acc_stderr": 0.03173284384294287, + "acc_norm": 0.2185430463576159, + "acc_norm_stderr": 0.03374235550425694 + }, + "hendrycksTest-computer_security": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.33, + "acc_norm_stderr": 0.047258156262526045 + }, + "hendrycksTest-global_facts": { + "acc": 0.2, + "acc_stderr": 0.040201512610368466, + "acc_norm": 0.22, + "acc_norm_stderr": 0.041633319989322695 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.84765625, + "likelihood_difference_stderr": 0.6465954210577143, + "pct_stereotype": 0.5138888888888888, + "pct_stereotype_stderr": 0.05931618532716554 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "hendrycksTest-nutrition": { + "acc": 0.2777777777777778, + "acc_stderr": 0.025646863097137908, + "acc_norm": 0.35947712418300654, + "acc_norm_stderr": 0.027475969910660952 + }, + "hendrycksTest-astronomy": { + "acc": 0.20394736842105263, + "acc_stderr": 0.0327900040631005, + "acc_norm": 0.3618421052631579, + "acc_norm_stderr": 0.03910525752849724 + }, + "hendrycksTest-college_physics": { + "acc": 0.21568627450980393, + "acc_stderr": 0.040925639582376536, + "acc_norm": 0.24509803921568626, + "acc_norm_stderr": 0.04280105837364395 + }, + "hendrycksTest-sociology": { + "acc": 0.29850746268656714, + "acc_stderr": 0.03235743789355043, + "acc_norm": 0.27860696517412936, + "acc_norm_stderr": 0.031700561834973086 + }, + "hendrycksTest-management": { + "acc": 0.23300970873786409, + "acc_stderr": 0.04185832598928312, + "acc_norm": 0.3106796116504854, + "acc_norm_stderr": 0.0458212416016155 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.24265644955300128, + "acc_stderr": 0.015329888940899858, + "acc_norm": 0.23754789272030652, + "acc_norm_stderr": 0.015218733046150195 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.23, + "acc_stderr": 0.04229525846816507, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + } + }, + "versions": { + "crows_pairs_english_gender": 0, + "hendrycksTest-college_chemistry": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-conceptual_physics": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-college_computer_science": 0, + "logiqa": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-virology": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-security_studies": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "crows_pairs_french": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-high_school_world_history": 0, + "wsc": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-high_school_mathematics": 0, + "crows_pairs_english": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-professional_medicine": 0, + "piqa": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-professional_accounting": 0, + "winogrande": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-prehistory": 0, + "arc_easy": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-high_school_statistics": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-elementary_mathematics": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-international_law": 0, + "lambada_openai": 0, + "arc_challenge": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-electrical_engineering": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-public_relations": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-high_school_geography": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-college_biology": 0, + "crows_pairs_french_socioeconomic": 0, + "crows_pairs_english_religion": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-professional_psychology": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-econometrics": 0, + "crows_pairs_french_religion": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-anatomy": 0, + "sciq": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-global_facts": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-management": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-abstract_algebra": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step133000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:5", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step143000.json b/data/checkpoint_eval/eleutherai_evals/step143000.json new file mode 100644 index 0000000000000000000000000000000000000000..1eb5d1f40ccba3eee5d802e73e63e550603f71a2 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step143000.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.950657894736842, + "likelihood_difference_stderr": 0.2574433600890588, + "pct_stereotype": 0.631578947368421, + "pct_stereotype_stderr": 0.03508771929824559 + }, + "hendrycksTest-global_facts": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.24, + "acc_norm_stderr": 0.042923469599092816 + }, + "hendrycksTest-formal_logic": { + "acc": 0.31746031746031744, + "acc_stderr": 0.04163453031302859, + "acc_norm": 0.29365079365079366, + "acc_norm_stderr": 0.040735243221471255 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.25722543352601157, + "acc_stderr": 0.02353292543104428, + "acc_norm": 0.3063583815028902, + "acc_norm_stderr": 0.024818350129436593 + }, + "hendrycksTest-college_biology": { + "acc": 0.2569444444444444, + "acc_stderr": 0.03653946969442099, + "acc_norm": 0.2708333333333333, + "acc_norm_stderr": 0.03716177437566017 + }, + "hendrycksTest-management": { + "acc": 0.1941747572815534, + "acc_stderr": 0.03916667762822583, + "acc_norm": 0.23300970873786409, + "acc_norm_stderr": 0.04185832598928315 + }, + "hendrycksTest-philosophy": { + "acc": 0.2540192926045016, + "acc_stderr": 0.024723861504771693, + "acc_norm": 0.2733118971061093, + "acc_norm_stderr": 0.02531176597542612 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-world_religions": { + "acc": 0.22807017543859648, + "acc_stderr": 0.032180937956023566, + "acc_norm": 0.2807017543859649, + "acc_norm_stderr": 0.034462962170884265 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.241263440860215, + "likelihood_difference_stderr": 0.5402840881668942, + "pct_stereotype": 0.8279569892473119, + "pct_stereotype_stderr": 0.039348528120618634 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 5.804258241758242, + "likelihood_difference_stderr": 0.47873767306009934, + "pct_stereotype": 0.7912087912087912, + "pct_stereotype_stderr": 0.04284305206509431 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.4914772727272725, + "likelihood_difference_stderr": 1.6384338230645699, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2907801418439716, + "acc_stderr": 0.027090664368353178, + "acc_norm": 0.29432624113475175, + "acc_norm_stderr": 0.027187127011503796 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2896551724137931, + "acc_stderr": 0.03780019230438014, + "acc_norm": 0.2896551724137931, + "acc_norm_stderr": 0.03780019230438014 + }, + "hendrycksTest-nutrition": { + "acc": 0.28104575163398693, + "acc_stderr": 0.02573885479781873, + "acc_norm": 0.3431372549019608, + "acc_norm_stderr": 0.027184498909941616 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.26053639846743293, + "acc_stderr": 0.015696008563807082, + "acc_norm": 0.2503192848020434, + "acc_norm_stderr": 0.015491088951494566 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.596590909090909, + "likelihood_difference_stderr": 0.7276689352721015, + "pct_stereotype": 0.48484848484848486, + "pct_stereotype_stderr": 0.06198888629778894 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.584584153543307, + "likelihood_difference_stderr": 0.17124663558058703, + "pct_stereotype": 0.452755905511811, + "pct_stereotype_stderr": 0.022106430541228055 + }, + "hendrycksTest-human_aging": { + "acc": 0.2556053811659193, + "acc_stderr": 0.029275891003969923, + "acc_norm": 0.21076233183856502, + "acc_norm_stderr": 0.027373095500540193 + }, + "hendrycksTest-business_ethics": { + "acc": 0.36, + "acc_stderr": 0.048241815132442176, + "acc_norm": 0.32, + "acc_norm_stderr": 0.04688261722621505 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.2324022346368715, + "acc_stderr": 0.014125968754673387, + "acc_norm": 0.2435754189944134, + "acc_norm_stderr": 0.014355911964767857 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.2565359477124183, + "acc_stderr": 0.017667841612378977, + "acc_norm": 0.28104575163398693, + "acc_norm_stderr": 0.018185218954318082 + }, + "hendrycksTest-marketing": { + "acc": 0.29914529914529914, + "acc_stderr": 0.02999695185834948, + "acc_norm": 0.2948717948717949, + "acc_norm_stderr": 0.029872577708891165 + }, + "hendrycksTest-sociology": { + "acc": 0.23880597014925373, + "acc_stderr": 0.03014777593540922, + "acc_norm": 0.27860696517412936, + "acc_norm_stderr": 0.031700561834973086 + }, + "logiqa": { + "acc": 0.21044546850998463, + "acc_stderr": 0.015988369488888737, + "acc_norm": 0.2565284178187404, + "acc_norm_stderr": 0.01712944332788756 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.2709677419354839, + "acc_stderr": 0.02528441611490016, + "acc_norm": 0.2967741935483871, + "acc_norm_stderr": 0.025988500792411898 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.24, + "acc_stderr": 0.04292346959909284, + "acc_norm": 0.37, + "acc_norm_stderr": 0.048523658709391 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.16, + "acc_stderr": 0.0368452949177471, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816506 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.396826171875, + "likelihood_difference_stderr": 0.2639919252147075, + "pct_stereotype": 0.484375, + "pct_stereotype_stderr": 0.027980952958187033 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 5.348641304347826, + "likelihood_difference_stderr": 0.542018501052962, + "pct_stereotype": 0.48695652173913045, + "pct_stereotype_stderr": 0.04681335351503156 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.16748768472906403, + "acc_stderr": 0.026273086047535397, + "acc_norm": 0.2561576354679803, + "acc_norm_stderr": 0.030712730070982592 + }, + "crows_pairs_english": { + "likelihood_difference": 3.730359272510435, + "likelihood_difference_stderr": 0.10184234185547236, + "pct_stereotype": 0.5193798449612403, + "pct_stereotype_stderr": 0.012204121667933781 + }, + "crows_pairs_french": { + "likelihood_difference": 5.56149373881932, + "likelihood_difference_stderr": 0.1424654102528186, + "pct_stereotype": 0.42695289206917114, + "pct_stereotype_stderr": 0.012082258834091222 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.2869198312236287, + "acc_stderr": 0.029443773022594693, + "acc_norm": 0.31223628691983124, + "acc_norm_stderr": 0.030165137867847004 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.26, + "acc_norm_stderr": 0.04408440022768079 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.26717557251908397, + "acc_stderr": 0.03880848301082394, + "acc_norm": 0.25190839694656486, + "acc_norm_stderr": 0.03807387116306086 + }, + "hendrycksTest-anatomy": { + "acc": 0.2518518518518518, + "acc_stderr": 0.03749850709174021, + "acc_norm": 0.23703703703703705, + "acc_norm_stderr": 0.03673731683969506 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.955078125, + "likelihood_difference_stderr": 0.3980155873439069, + "pct_stereotype": 0.5833333333333334, + "pct_stereotype_stderr": 0.058509124791617455 + }, + "lambada_openai": { + "ppl": 142.42891015470678, + "ppl_stderr": 6.043810708201551, + "acc": 0.18455268775470599, + "acc_stderr": 0.00540468283118203 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.5283564814814814, + "likelihood_difference_stderr": 0.26698531590286084, + "pct_stereotype": 0.4074074074074074, + "pct_stereotype_stderr": 0.03350991604696043 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.2549019607843137, + "acc_stderr": 0.030587591351604246, + "acc_norm": 0.27941176470588236, + "acc_norm_stderr": 0.03149328104507956 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.2605042016806723, + "acc_stderr": 0.028510251512341933, + "acc_norm": 0.35714285714285715, + "acc_norm_stderr": 0.031124619309328177 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.814945652173913, + "likelihood_difference_stderr": 0.26747469667074003, + "pct_stereotype": 0.3391304347826087, + "pct_stereotype_stderr": 0.02209708145176117 + }, + "hendrycksTest-college_physics": { + "acc": 0.20588235294117646, + "acc_stderr": 0.04023382273617747, + "acc_norm": 0.2549019607843137, + "acc_norm_stderr": 0.04336432707993177 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.641744548286605, + "likelihood_difference_stderr": 0.2415529745762916, + "pct_stereotype": 0.5015576323987538, + "pct_stereotype_stderr": 0.02795071408867036 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816506 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.24074074074074073, + "acc_stderr": 0.0413311944024384, + "acc_norm": 0.37037037037037035, + "acc_norm_stderr": 0.04668408033024931 + }, + "hendrycksTest-international_law": { + "acc": 0.2066115702479339, + "acc_stderr": 0.036959801280988226, + "acc_norm": 0.4380165289256198, + "acc_norm_stderr": 0.045291468044357915 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.739864864864865, + "likelihood_difference_stderr": 0.4263788133283603, + "pct_stereotype": 0.6396396396396397, + "pct_stereotype_stderr": 0.04577621167070315 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.602430555555555, + "likelihood_difference_stderr": 0.5176263692232941, + "pct_stereotype": 0.4, + "pct_stereotype_stderr": 0.051929078688949845 + }, + "hendrycksTest-security_studies": { + "acc": 0.363265306122449, + "acc_stderr": 0.030789051139030802, + "acc_norm": 0.31020408163265306, + "acc_norm_stderr": 0.029613459872484375 + }, + "hendrycksTest-machine_learning": { + "acc": 0.25, + "acc_stderr": 0.04109974682633932, + "acc_norm": 0.22321428571428573, + "acc_norm_stderr": 0.039523019677025116 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 3.8822115384615383, + "likelihood_difference_stderr": 1.0329310420047324, + "pct_stereotype": 0.38461538461538464, + "pct_stereotype_stderr": 0.1404416814115811 + }, + "sciq": { + "acc": 0.601, + "acc_stderr": 0.015493193313162906, + "acc_norm": 0.552, + "acc_norm_stderr": 0.015733516566347833 + }, + "hendrycksTest-prehistory": { + "acc": 0.2962962962962963, + "acc_stderr": 0.02540719779889016, + "acc_norm": 0.25, + "acc_norm_stderr": 0.02409347123262133 + }, + "hendrycksTest-econometrics": { + "acc": 0.2894736842105263, + "acc_stderr": 0.04266339443159394, + "acc_norm": 0.21929824561403508, + "acc_norm_stderr": 0.03892431106518754 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 5.561702806122449, + "likelihood_difference_stderr": 0.4452498998491719, + "pct_stereotype": 0.45408163265306123, + "pct_stereotype_stderr": 0.035654431417332814 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.24242424242424243, + "acc_stderr": 0.03346409881055953, + "acc_norm": 0.3151515151515151, + "acc_norm_stderr": 0.0362773057502241 + }, + "hendrycksTest-astronomy": { + "acc": 0.26973684210526316, + "acc_stderr": 0.036117805602848975, + "acc_norm": 0.375, + "acc_norm_stderr": 0.039397364351956274 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.22699386503067484, + "acc_stderr": 0.03291099578615769, + "acc_norm": 0.25153374233128833, + "acc_norm_stderr": 0.034089978868575295 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.2251655629139073, + "acc_stderr": 0.03410435282008937, + "acc_norm": 0.2582781456953642, + "acc_norm_stderr": 0.035737053147634576 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 8.138463438735178, + "likelihood_difference_stderr": 0.4545360074730252, + "pct_stereotype": 0.2845849802371542, + "pct_stereotype_stderr": 0.02842397052208522 + }, + "hendrycksTest-virology": { + "acc": 0.2710843373493976, + "acc_stderr": 0.03460579907553027, + "acc_norm": 0.24096385542168675, + "acc_norm_stderr": 0.033293941190735296 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.25906735751295334, + "acc_stderr": 0.031618779179354094, + "acc_norm": 0.27461139896373055, + "acc_norm_stderr": 0.03221024508041154 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.389182692307692, + "likelihood_difference_stderr": 0.6517633770952715, + "pct_stereotype": 0.5692307692307692, + "pct_stereotype_stderr": 0.061897988228581086 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.18888888888888888, + "acc_stderr": 0.023865318862285302, + "acc_norm": 0.22962962962962963, + "acc_norm_stderr": 0.02564410863926762 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.26238532110091745, + "acc_stderr": 0.018861885021534738, + "acc_norm": 0.22935779816513763, + "acc_norm_stderr": 0.018025349724618688 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.962239583333333, + "likelihood_difference_stderr": 0.694376691160675, + "pct_stereotype": 0.5138888888888888, + "pct_stereotype_stderr": 0.05931618532716554 + }, + "hendrycksTest-college_medicine": { + "acc": 0.2658959537572254, + "acc_stderr": 0.0336876293225943, + "acc_norm": 0.2832369942196532, + "acc_norm_stderr": 0.03435568056047873 + }, + "hendrycksTest-professional_law": { + "acc": 0.2457627118644068, + "acc_stderr": 0.01099615663514269, + "acc_norm": 0.2790091264667536, + "acc_norm_stderr": 0.011455208832803546 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.26, + "acc_stderr": 0.044084400227680794, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.25132275132275134, + "acc_stderr": 0.022340482339643895, + "acc_norm": 0.23544973544973544, + "acc_norm_stderr": 0.02185150982203172 + }, + "arc_easy": { + "acc": 0.37373737373737376, + "acc_stderr": 0.009927267058259621, + "acc_norm": 0.3501683501683502, + "acc_norm_stderr": 0.009788295410093142 + }, + "arc_challenge": { + "acc": 0.18088737201365188, + "acc_stderr": 0.01124857446740701, + "acc_norm": 0.22098976109215018, + "acc_norm_stderr": 0.012124929206818258 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.605254120879121, + "likelihood_difference_stderr": 0.2698048849736164, + "pct_stereotype": 0.5054945054945055, + "pct_stereotype_stderr": 0.05270144531112881 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2297872340425532, + "acc_stderr": 0.02750175294441242, + "acc_norm": 0.1829787234042553, + "acc_norm_stderr": 0.025276041000449966 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "acc_norm": 0.32, + "acc_norm_stderr": 0.046882617226215034 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.21794871794871795, + "acc_stderr": 0.020932445774463175, + "acc_norm": 0.2794871794871795, + "acc_norm_stderr": 0.022752388839776826 + }, + "winogrande": { + "acc": 0.5280189423835833, + "acc_stderr": 0.014030404213405777 + }, + "hendrycksTest-public_relations": { + "acc": 0.3, + "acc_stderr": 0.04389311454644286, + "acc_norm": 0.19090909090909092, + "acc_norm_stderr": 0.03764425585984924 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.2676767676767677, + "acc_stderr": 0.031544498882702866, + "acc_norm": 0.2777777777777778, + "acc_norm_stderr": 0.03191178226713547 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.2339622641509434, + "acc_stderr": 0.02605529690115292, + "acc_norm": 0.2943396226415094, + "acc_norm_stderr": 0.028049186315695245 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.2867647058823529, + "acc_stderr": 0.027472274473233818, + "acc_norm": 0.25735294117647056, + "acc_norm_stderr": 0.026556519470041524 + }, + "piqa": { + "acc": 0.5946681175190425, + "acc_stderr": 0.011454816387346764, + "acc_norm": 0.5914036996735582, + "acc_norm_stderr": 0.01146924038724515 + }, + "hendrycksTest-computer_security": { + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "acc_norm": 0.27, + "acc_norm_stderr": 0.0446196043338474 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.2037037037037037, + "acc_stderr": 0.027467401804057996, + "acc_norm": 0.2361111111111111, + "acc_norm_stderr": 0.02896370257079103 + } + }, + "versions": { + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-management": 0, + "hendrycksTest-philosophy": 0, + "wsc": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_english_sexual_orientation": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-college_chemistry": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-miscellaneous": 0, + "crows_pairs_french_disability": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-sociology": 0, + "logiqa": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-college_mathematics": 0, + "crows_pairs_english_gender": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-high_school_chemistry": 0, + "crows_pairs_english": 0, + "crows_pairs_french": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-anatomy": 0, + "crows_pairs_english_physical_appearance": 0, + "lambada_openai": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-high_school_microeconomics": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-college_physics": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-international_law": 0, + "crows_pairs_english_religion": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-machine_learning": 0, + "crows_pairs_french_autre": 0, + "sciq": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-econometrics": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-high_school_physics": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-high_school_psychology": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-elementary_mathematics": 0, + "arc_easy": 0, + "arc_challenge": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "winogrande": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-professional_medicine": 0, + "piqa": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-high_school_statistics": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step143000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:7", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step16.json b/data/checkpoint_eval/eleutherai_evals/step16.json new file mode 100644 index 0000000000000000000000000000000000000000..39a2a773b82981692b5bd8a09ce3bdc5eccae12a --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step16.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_english_religion": { + "likelihood_difference": 5.7657657657657655, + "likelihood_difference_stderr": 0.7051443071103357, + "pct_stereotype": 0.5135135135135135, + "pct_stereotype_stderr": 0.04765571461988585 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.16666666666666666, + "acc_stderr": 0.036028141763926456, + "acc_norm": 0.26851851851851855, + "acc_norm_stderr": 0.04284467968052191 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.22592592592592592, + "acc_stderr": 0.02549753263960955, + "acc_norm": 0.2740740740740741, + "acc_norm_stderr": 0.027195934804085626 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.20725388601036268, + "acc_stderr": 0.02925282329180363, + "acc_norm": 0.25906735751295334, + "acc_norm_stderr": 0.031618779179354094 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2366412213740458, + "acc_stderr": 0.03727673575596918, + "acc_norm": 0.2824427480916031, + "acc_norm_stderr": 0.03948406125768362 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 5.946875, + "likelihood_difference_stderr": 0.5419079991233648, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.036369648372665396 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.22794117647058823, + "acc_stderr": 0.025483081468029804, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.026799562024887685 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.22486772486772486, + "acc_stderr": 0.021502096078229147, + "acc_norm": 0.23544973544973544, + "acc_norm_stderr": 0.021851509822031708 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.24770642201834864, + "acc_stderr": 0.018508143602547798, + "acc_norm": 0.25504587155963304, + "acc_norm_stderr": 0.018688500856535843 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 14.919507575757576, + "likelihood_difference_stderr": 1.4664292625655948, + "pct_stereotype": 0.3787878787878788, + "pct_stereotype_stderr": 0.06016741025240241 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.2474747474747475, + "acc_stderr": 0.030746300742124488, + "acc_norm": 0.25757575757575757, + "acc_norm_stderr": 0.03115626951964684 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "acc_norm": 0.18, + "acc_norm_stderr": 0.03861229196653695 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.2, + "acc_stderr": 0.04020151261036846, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542128 + }, + "hendrycksTest-computer_security": { + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206845 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "acc_norm": 0.2, + "acc_norm_stderr": 0.04020151261036845 + }, + "hendrycksTest-formal_logic": { + "acc": 0.2698412698412698, + "acc_stderr": 0.03970158273235172, + "acc_norm": 0.2619047619047619, + "acc_norm_stderr": 0.039325376803928724 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.2138728323699422, + "acc_stderr": 0.022075709251757177, + "acc_norm": 0.20809248554913296, + "acc_norm_stderr": 0.0218552552634218 + }, + "hendrycksTest-professional_law": { + "acc": 0.2333767926988266, + "acc_stderr": 0.010803108481179097, + "acc_norm": 0.23728813559322035, + "acc_norm_stderr": 0.01086543669078027 + }, + "hendrycksTest-prehistory": { + "acc": 0.2654320987654321, + "acc_stderr": 0.024569223600460852, + "acc_norm": 0.22839506172839505, + "acc_norm_stderr": 0.023358211840626267 + }, + "winogrande": { + "acc": 0.48224151539068666, + "acc_stderr": 0.014043619596174964 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.18, + "acc_stderr": 0.038612291966536955, + "acc_norm": 0.18, + "acc_norm_stderr": 0.03861229196653696 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2681992337164751, + "acc_stderr": 0.015842430835269435, + "acc_norm": 0.26436781609195403, + "acc_norm_stderr": 0.015769984840690518 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.2361111111111111, + "acc_stderr": 0.028963702570791037, + "acc_norm": 0.2824074074074074, + "acc_norm_stderr": 0.030701372111510934 + }, + "hendrycksTest-business_ethics": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.26, + "acc_norm_stderr": 0.04408440022768079 + }, + "sciq": { + "acc": 0.205, + "acc_stderr": 0.012772554096113121, + "acc_norm": 0.205, + "acc_norm_stderr": 0.012772554096113123 + }, + "hendrycksTest-marketing": { + "acc": 0.23076923076923078, + "acc_stderr": 0.027601921381417607, + "acc_norm": 0.2564102564102564, + "acc_norm_stderr": 0.028605953702004243 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2, + "acc_stderr": 0.0333333333333333, + "acc_norm": 0.2, + "acc_norm_stderr": 0.03333333333333331 + }, + "hendrycksTest-global_facts": { + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "acc_norm": 0.26, + "acc_norm_stderr": 0.044084400227680794 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 6.262586805555555, + "likelihood_difference_stderr": 0.8386766732679737, + "pct_stereotype": 0.5694444444444444, + "pct_stereotype_stderr": 0.05876396677084613 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 9.763833992094861, + "likelihood_difference_stderr": 0.5454943873296275, + "pct_stereotype": 0.5533596837944664, + "pct_stereotype_stderr": 0.03131716554414947 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.21, + "acc_stderr": 0.04093601807403325, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.792613636363637, + "likelihood_difference_stderr": 3.094109462379434, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "crows_pairs_french_age": { + "likelihood_difference": 5.9534722222222225, + "likelihood_difference_stderr": 0.7502305104509913, + "pct_stereotype": 0.6666666666666666, + "pct_stereotype_stderr": 0.04996877926639073 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 7.8625, + "likelihood_difference_stderr": 1.2300665742321708, + "pct_stereotype": 0.6, + "pct_stereotype_stderr": 0.06123724356957946 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 9.584782608695653, + "likelihood_difference_stderr": 0.3782157420364811, + "pct_stereotype": 0.6978260869565217, + "pct_stereotype_stderr": 0.021433630616234406 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.2230769230769231, + "acc_stderr": 0.021107730127243988, + "acc_norm": 0.23846153846153847, + "acc_norm_stderr": 0.021606294494647734 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.18565400843881857, + "acc_stderr": 0.025310495376944856, + "acc_norm": 0.20253164556962025, + "acc_norm_stderr": 0.02616056824660147 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.498655913978495, + "likelihood_difference_stderr": 0.7803814003203932, + "pct_stereotype": 0.45161290322580644, + "pct_stereotype_stderr": 0.051883930752016603 + }, + "hendrycksTest-college_medicine": { + "acc": 0.20809248554913296, + "acc_stderr": 0.030952890217749884, + "acc_norm": 0.24277456647398843, + "acc_norm_stderr": 0.0326926380614177 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.18787878787878787, + "acc_stderr": 0.03050193405942914, + "acc_norm": 0.20606060606060606, + "acc_norm_stderr": 0.031584153240477086 + }, + "hendrycksTest-philosophy": { + "acc": 0.22186495176848875, + "acc_stderr": 0.02359885829286305, + "acc_norm": 0.2861736334405145, + "acc_norm_stderr": 0.025670259242188943 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 10.22482638888889, + "likelihood_difference_stderr": 1.3720668559778915, + "pct_stereotype": 0.4861111111111111, + "pct_stereotype_stderr": 0.05931618532716555 + }, + "crows_pairs_english": { + "likelihood_difference": 5.680251192605843, + "likelihood_difference_stderr": 0.1962056921179939, + "pct_stereotype": 0.4883720930232558, + "pct_stereotype_stderr": 0.01220999609506964 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.2019704433497537, + "acc_stderr": 0.02824735012218027, + "acc_norm": 0.21182266009852216, + "acc_norm_stderr": 0.028748983689941075 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.2, + "acc_norm_stderr": 0.040201512610368445 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.17647058823529413, + "acc_stderr": 0.026756401538078955, + "acc_norm": 0.2549019607843137, + "acc_norm_stderr": 0.030587591351604257 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.20967741935483872, + "acc_stderr": 0.02315787934908352, + "acc_norm": 0.22258064516129034, + "acc_norm_stderr": 0.023664216671642518 + }, + "piqa": { + "acc": 0.5266594124047879, + "acc_stderr": 0.011649229994347386, + "acc_norm": 0.5233949945593036, + "acc_norm_stderr": 0.011653047155927796 + }, + "hendrycksTest-human_aging": { + "acc": 0.27802690582959644, + "acc_stderr": 0.03006958487449403, + "acc_norm": 0.27802690582959644, + "acc_norm_stderr": 0.03006958487449403 + }, + "hendrycksTest-econometrics": { + "acc": 0.22807017543859648, + "acc_stderr": 0.03947152782669415, + "acc_norm": 0.3157894736842105, + "acc_norm_stderr": 0.04372748290278008 + }, + "hendrycksTest-sociology": { + "acc": 0.26865671641791045, + "acc_stderr": 0.031343283582089536, + "acc_norm": 0.263681592039801, + "acc_norm_stderr": 0.03115715086935557 + }, + "hendrycksTest-astronomy": { + "acc": 0.18421052631578946, + "acc_stderr": 0.0315469804508223, + "acc_norm": 0.23026315789473684, + "acc_norm_stderr": 0.03426059424403165 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.64010989010989, + "likelihood_difference_stderr": 0.6494737736897132, + "pct_stereotype": 0.6043956043956044, + "pct_stereotype_stderr": 0.05154303032773001 + }, + "logiqa": { + "acc": 0.2227342549923195, + "acc_stderr": 0.016320054046165128, + "acc_norm": 0.2519201228878648, + "acc_norm_stderr": 0.017027415657021122 + }, + "crows_pairs_french": { + "likelihood_difference": 10.133110651460942, + "likelihood_difference_stderr": 0.23305057763604026, + "pct_stereotype": 0.5915324985092427, + "pct_stereotype_stderr": 0.012006904380989914 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.18487394957983194, + "acc_stderr": 0.025215992877954205, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.028657491285071973 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 14.914835164835164, + "likelihood_difference_stderr": 1.0458321095115632, + "pct_stereotype": 0.7912087912087912, + "pct_stereotype_stderr": 0.04284305206509431 + }, + "hendrycksTest-public_relations": { + "acc": 0.2545454545454545, + "acc_stderr": 0.041723430387053825, + "acc_norm": 0.2, + "acc_norm_stderr": 0.03831305140884603 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.18490566037735848, + "acc_stderr": 0.02389335183446432, + "acc_norm": 0.29056603773584905, + "acc_norm_stderr": 0.027943219989337142 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 7.616043613707165, + "likelihood_difference_stderr": 0.453671327786389, + "pct_stereotype": 0.5482866043613707, + "pct_stereotype_stderr": 0.027820204204815787 + }, + "arc_challenge": { + "acc": 0.2090443686006826, + "acc_stderr": 0.011882746987406457, + "acc_norm": 0.24829351535836178, + "acc_norm_stderr": 0.01262491286808977 + }, + "hendrycksTest-college_physics": { + "acc": 0.21568627450980393, + "acc_stderr": 0.04092563958237656, + "acc_norm": 0.22549019607843138, + "acc_norm_stderr": 0.04158307533083286 + }, + "hendrycksTest-world_religions": { + "acc": 0.19883040935672514, + "acc_stderr": 0.030611116557432535, + "acc_norm": 0.25146198830409355, + "acc_norm_stderr": 0.033275044238468436 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 12.976283482142858, + "likelihood_difference_stderr": 0.8478950106991682, + "pct_stereotype": 0.4387755102040816, + "pct_stereotype_stderr": 0.03553629865790393 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 7.144230769230769, + "likelihood_difference_stderr": 2.005306269861725, + "pct_stereotype": 0.3076923076923077, + "pct_stereotype_stderr": 0.13323467750529824 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.15337423312883436, + "acc_stderr": 0.02831160144143859, + "acc_norm": 0.22699386503067484, + "acc_norm_stderr": 0.03291099578615769 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 4.92646484375, + "likelihood_difference_stderr": 0.4975177732945778, + "pct_stereotype": 0.503125, + "pct_stereotype_stderr": 0.027994078772422815 + }, + "hendrycksTest-international_law": { + "acc": 0.10743801652892562, + "acc_stderr": 0.028268812192540637, + "acc_norm": 0.23140495867768596, + "acc_norm_stderr": 0.038498560987940904 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 6.327112268518518, + "likelihood_difference_stderr": 0.5322781195091362, + "pct_stereotype": 0.4444444444444444, + "pct_stereotype_stderr": 0.03388857118502325 + }, + "hendrycksTest-virology": { + "acc": 0.16265060240963855, + "acc_stderr": 0.0287302378926138, + "acc_norm": 0.20481927710843373, + "acc_norm_stderr": 0.03141784291663926 + }, + "hendrycksTest-college_biology": { + "acc": 0.2569444444444444, + "acc_stderr": 0.03653946969442099, + "acc_norm": 0.2708333333333333, + "acc_norm_stderr": 0.03716177437566017 + }, + "hendrycksTest-machine_learning": { + "acc": 0.25, + "acc_stderr": 0.04109974682633932, + "acc_norm": 0.30357142857142855, + "acc_norm_stderr": 0.04364226155841044 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.1986754966887417, + "acc_stderr": 0.03257847384436777, + "acc_norm": 0.31788079470198677, + "acc_norm_stderr": 0.038020397601079024 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "acc_norm": 0.34, + "acc_norm_stderr": 0.047609522856952344 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.29432624113475175, + "acc_stderr": 0.027187127011503803, + "acc_norm": 0.2695035460992908, + "acc_norm_stderr": 0.026469036818590634 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 12.339673913043478, + "likelihood_difference_stderr": 0.968029517405933, + "pct_stereotype": 0.6347826086956522, + "pct_stereotype_stderr": 0.04509577025262067 + }, + "lambada_openai": { + "ppl": 3526086.6936456356, + "ppl_stderr": 340136.02871194785, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2851063829787234, + "acc_stderr": 0.029513196625539355, + "acc_norm": 0.2553191489361702, + "acc_norm_stderr": 0.02850485647051418 + }, + "hendrycksTest-anatomy": { + "acc": 0.23703703703703705, + "acc_stderr": 0.03673731683969506, + "acc_norm": 0.2962962962962963, + "acc_norm_stderr": 0.03944624162501117 + }, + "arc_easy": { + "acc": 0.26262626262626265, + "acc_stderr": 0.009029861776763754, + "acc_norm": 0.2516835016835017, + "acc_norm_stderr": 0.008905088235948768 + }, + "hendrycksTest-security_studies": { + "acc": 0.3183673469387755, + "acc_stderr": 0.02982253379398205, + "acc_norm": 0.19591836734693877, + "acc_norm_stderr": 0.025409301953225678 + }, + "wsc": { + "acc": 0.6346153846153846, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.2173202614379085, + "acc_stderr": 0.016684820929148622, + "acc_norm": 0.23039215686274508, + "acc_norm_stderr": 0.017035229258034044 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 5.796136811023622, + "likelihood_difference_stderr": 0.35545568796836857, + "pct_stereotype": 0.44881889763779526, + "pct_stereotype_stderr": 0.02208913692163594 + }, + "hendrycksTest-management": { + "acc": 0.21359223300970873, + "acc_stderr": 0.04058042015646033, + "acc_norm": 0.27184466019417475, + "acc_norm_stderr": 0.044052680241409216 + }, + "hendrycksTest-nutrition": { + "acc": 0.1830065359477124, + "acc_stderr": 0.02214076751288095, + "acc_norm": 0.25163398692810457, + "acc_norm_stderr": 0.0248480182638752 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + } + }, + "versions": { + "crows_pairs_english_religion": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-high_school_psychology": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-prehistory": 0, + "winogrande": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-business_ethics": 0, + "sciq": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-global_facts": 0, + "crows_pairs_english_physical_appearance": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-college_chemistry": 0, + "crows_pairs_english_autre": 0, + "crows_pairs_french_age": 0, + "crows_pairs_english_disability": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-high_school_world_history": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-philosophy": 0, + "crows_pairs_french_physical_appearance": 0, + "crows_pairs_english": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-high_school_biology": 0, + "piqa": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-astronomy": 0, + "crows_pairs_english_age": 0, + "logiqa": 0, + "crows_pairs_french": 0, + "hendrycksTest-high_school_microeconomics": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_french_gender": 0, + "arc_challenge": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_french_socioeconomic": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-logical_fallacies": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-international_law": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-professional_accounting": 0, + "crows_pairs_french_religion": 0, + "lambada_openai": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-anatomy": 0, + "arc_easy": 0, + "hendrycksTest-security_studies": 0, + "wsc": 0, + "hendrycksTest-professional_psychology": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-management": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-moral_scenarios": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step16", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step2.json b/data/checkpoint_eval/eleutherai_evals/step2.json new file mode 100644 index 0000000000000000000000000000000000000000..27ccc5011f205be32c2ee213dc3f6baa4a44bf19 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step2.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_english_disability": { + "likelihood_difference": 7.950961538461539, + "likelihood_difference_stderr": 1.2464549281452417, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.06231481440776789 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 14.897664835164836, + "likelihood_difference_stderr": 1.023996890557528, + "pct_stereotype": 0.8021978021978022, + "pct_stereotype_stderr": 0.04198895203196222 + }, + "logiqa": { + "acc": 0.22887864823348694, + "acc_stderr": 0.01647810727631327, + "acc_norm": 0.24731182795698925, + "acc_norm_stderr": 0.016922842446712393 + }, + "sciq": { + "acc": 0.194, + "acc_stderr": 0.012510816141264368, + "acc_norm": 0.216, + "acc_norm_stderr": 0.013019735539307789 + }, + "arc_easy": { + "acc": 0.27104377104377103, + "acc_stderr": 0.009120919741760602, + "acc_norm": 0.2516835016835017, + "acc_norm_stderr": 0.008905088235948768 + }, + "hendrycksTest-business_ethics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-world_religions": { + "acc": 0.19298245614035087, + "acc_stderr": 0.03026745755489847, + "acc_norm": 0.23391812865497075, + "acc_norm_stderr": 0.03246721765117825 + }, + "hendrycksTest-philosophy": { + "acc": 0.22508038585209003, + "acc_stderr": 0.023720088516179027, + "acc_norm": 0.2958199356913183, + "acc_norm_stderr": 0.025922371788818784 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.842741935483871, + "likelihood_difference_stderr": 0.7853109235496362, + "pct_stereotype": 0.44086021505376344, + "pct_stereotype_stderr": 0.051762678118979284 + }, + "crows_pairs_english": { + "likelihood_difference": 5.816692754919499, + "likelihood_difference_stderr": 0.2005617516277463, + "pct_stereotype": 0.45915324985092426, + "pct_stereotype_stderr": 0.012172476264191387 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.17721518987341772, + "acc_stderr": 0.024856364184503217, + "acc_norm": 0.21518987341772153, + "acc_norm_stderr": 0.02675082699467616 + }, + "hendrycksTest-international_law": { + "acc": 0.08264462809917356, + "acc_stderr": 0.02513538235660422, + "acc_norm": 0.2066115702479339, + "acc_norm_stderr": 0.03695980128098824 + }, + "hendrycksTest-machine_learning": { + "acc": 0.3125, + "acc_stderr": 0.043994650575715215, + "acc_norm": 0.30357142857142855, + "acc_norm_stderr": 0.04364226155841044 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2553191489361702, + "acc_stderr": 0.0285048564705142, + "acc_norm": 0.25957446808510637, + "acc_norm_stderr": 0.02865917937429232 + }, + "arc_challenge": { + "acc": 0.21416382252559726, + "acc_stderr": 0.011988383205966508, + "acc_norm": 0.24658703071672355, + "acc_norm_stderr": 0.01259572626879014 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 12.441304347826087, + "likelihood_difference_stderr": 0.9700478612500396, + "pct_stereotype": 0.6086956521739131, + "pct_stereotype_stderr": 0.045709346351117126 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 5.947957677165355, + "likelihood_difference_stderr": 0.3647056359584575, + "pct_stereotype": 0.3641732283464567, + "pct_stereotype_stderr": 0.021370733739311764 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816506 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2503192848020434, + "acc_stderr": 0.01549108895149458, + "acc_norm": 0.26436781609195403, + "acc_norm_stderr": 0.015769984840690518 + }, + "hendrycksTest-college_medicine": { + "acc": 0.20809248554913296, + "acc_stderr": 0.03095289021774988, + "acc_norm": 0.2658959537572254, + "acc_norm_stderr": 0.03368762932259431 + }, + "hendrycksTest-global_facts": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "acc_norm": 0.27, + "acc_norm_stderr": 0.044619604333847394 + }, + "lambada_openai": { + "ppl": 3683856.625077963, + "ppl_stderr": 360078.72745232895, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "winogrande": { + "acc": 0.4980268350434096, + "acc_stderr": 0.014052376259225632 + }, + "hendrycksTest-marketing": { + "acc": 0.23504273504273504, + "acc_stderr": 0.027778835904935437, + "acc_norm": 0.2564102564102564, + "acc_norm_stderr": 0.028605953702004243 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.18543046357615894, + "acc_stderr": 0.031732843842942865, + "acc_norm": 0.31125827814569534, + "acc_norm_stderr": 0.03780445850526733 + }, + "hendrycksTest-nutrition": { + "acc": 0.17973856209150327, + "acc_stderr": 0.021986032182064148, + "acc_norm": 0.238562091503268, + "acc_norm_stderr": 0.024404394928087877 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.1574074074074074, + "acc_stderr": 0.03520703990517965, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.042365112580946315 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 13.086854272959183, + "likelihood_difference_stderr": 0.8661141987300517, + "pct_stereotype": 0.413265306122449, + "pct_stereotype_stderr": 0.035262902194360866 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.26, + "acc_stderr": 0.04408440022768079, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036624 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.19310344827586207, + "acc_stderr": 0.03289445522127401, + "acc_norm": 0.19310344827586207, + "acc_norm_stderr": 0.03289445522127401 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.18, + "acc_stderr": 0.03861229196653695, + "acc_norm": 0.17, + "acc_norm_stderr": 0.03775251680686371 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 6.284090909090909, + "likelihood_difference_stderr": 3.185464631314461, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-management": { + "acc": 0.2524271844660194, + "acc_stderr": 0.04301250399690879, + "acc_norm": 0.2912621359223301, + "acc_norm_stderr": 0.044986763205729245 + }, + "hendrycksTest-security_studies": { + "acc": 0.3142857142857143, + "acc_stderr": 0.02971932942241746, + "acc_norm": 0.19183673469387755, + "acc_norm_stderr": 0.025206963154225378 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2595419847328244, + "acc_stderr": 0.03844876139785271, + "acc_norm": 0.2595419847328244, + "acc_norm_stderr": 0.03844876139785271 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 6.443142361111111, + "likelihood_difference_stderr": 0.539056794039704, + "pct_stereotype": 0.44907407407407407, + "pct_stereotype_stderr": 0.03392238405321617 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.2275132275132275, + "acc_stderr": 0.021591269407823785, + "acc_norm": 0.23544973544973544, + "acc_norm_stderr": 0.021851509822031715 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.25504587155963304, + "acc_stderr": 0.018688500856535853, + "acc_norm": 0.24587155963302754, + "acc_norm_stderr": 0.018461940968708457 + }, + "hendrycksTest-anatomy": { + "acc": 0.21481481481481482, + "acc_stderr": 0.03547854198560823, + "acc_norm": 0.2962962962962963, + "acc_norm_stderr": 0.03944624162501117 + }, + "hendrycksTest-college_physics": { + "acc": 0.23529411764705882, + "acc_stderr": 0.04220773659171452, + "acc_norm": 0.22549019607843138, + "acc_norm_stderr": 0.04158307533083286 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.23737373737373738, + "acc_stderr": 0.03031371053819888, + "acc_norm": 0.2727272727272727, + "acc_norm_stderr": 0.03173071239071724 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 5.01279296875, + "likelihood_difference_stderr": 0.5134355620258765, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.027994625547792713 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.1656441717791411, + "acc_stderr": 0.029208296231259104, + "acc_norm": 0.2392638036809816, + "acc_norm_stderr": 0.033519538795212696 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.17575757575757575, + "acc_stderr": 0.02972094300622445, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.030117688929503585 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.25462962962962965, + "acc_stderr": 0.029711275860005354, + "acc_norm": 0.2916666666666667, + "acc_norm_stderr": 0.030998666304560534 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 14.813446969696969, + "likelihood_difference_stderr": 1.4757253537933575, + "pct_stereotype": 0.3484848484848485, + "pct_stereotype_stderr": 0.0591013677911929 + }, + "wsc": { + "acc": 0.5961538461538461, + "acc_stderr": 0.048346889526540184 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.23410404624277456, + "acc_stderr": 0.022797110278071138, + "acc_norm": 0.21098265895953758, + "acc_norm_stderr": 0.021966309947043117 + }, + "piqa": { + "acc": 0.5228509249183896, + "acc_stderr": 0.01165363483240117, + "acc_norm": 0.5212187159956474, + "acc_norm_stderr": 0.01165531473228886 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.21761658031088082, + "acc_stderr": 0.029778663037752964, + "acc_norm": 0.24352331606217617, + "acc_norm_stderr": 0.030975436386845436 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 5.919763513513513, + "likelihood_difference_stderr": 0.7154224085059995, + "pct_stereotype": 0.5225225225225225, + "pct_stereotype_stderr": 0.04762473917649626 + }, + "hendrycksTest-sociology": { + "acc": 0.25870646766169153, + "acc_stderr": 0.030965903123573044, + "acc_norm": 0.2736318407960199, + "acc_norm_stderr": 0.03152439186555402 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.20967741935483872, + "acc_stderr": 0.02315787934908352, + "acc_norm": 0.22258064516129034, + "acc_norm_stderr": 0.02366421667164251 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "acc_norm": 0.34, + "acc_norm_stderr": 0.047609522856952344 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.22660098522167488, + "acc_stderr": 0.029454863835293, + "acc_norm": 0.21674876847290642, + "acc_norm_stderr": 0.028990331252516235 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.2222222222222222, + "acc_stderr": 0.01681902837573638, + "acc_norm": 0.23366013071895425, + "acc_norm_stderr": 0.017119158496044503 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 6.058059210526316, + "likelihood_difference_stderr": 0.5512612681732508, + "pct_stereotype": 0.47368421052631576, + "pct_stereotype_stderr": 0.03631923996538703 + }, + "hendrycksTest-astronomy": { + "acc": 0.18421052631578946, + "acc_stderr": 0.031546980450822305, + "acc_norm": 0.23684210526315788, + "acc_norm_stderr": 0.03459777606810534 + }, + "hendrycksTest-econometrics": { + "acc": 0.2543859649122807, + "acc_stderr": 0.04096985139843669, + "acc_norm": 0.30701754385964913, + "acc_norm_stderr": 0.0433913832257986 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 10.164930555555555, + "likelihood_difference_stderr": 1.3628801155784467, + "pct_stereotype": 0.5138888888888888, + "pct_stereotype_stderr": 0.05931618532716555 + }, + "crows_pairs_french_age": { + "likelihood_difference": 5.870138888888889, + "likelihood_difference_stderr": 0.7718652798506598, + "pct_stereotype": 0.6444444444444445, + "pct_stereotype_stderr": 0.05074011803597719 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 8.873641304347826, + "likelihood_difference_stderr": 0.3747500277540656, + "pct_stereotype": 0.6413043478260869, + "pct_stereotype_stderr": 0.02238663434141095 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 6.980769230769231, + "likelihood_difference_stderr": 1.9247068413265722, + "pct_stereotype": 0.38461538461538464, + "pct_stereotype_stderr": 0.1404416814115811 + }, + "hendrycksTest-professional_law": { + "acc": 0.23272490221642764, + "acc_stderr": 0.010792595553888486, + "acc_norm": 0.242503259452412, + "acc_norm_stderr": 0.010946570966348788 + }, + "hendrycksTest-public_relations": { + "acc": 0.2545454545454545, + "acc_stderr": 0.041723430387053825, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.03694284335337798 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.24632352941176472, + "acc_stderr": 0.02617343857052, + "acc_norm": 0.27941176470588236, + "acc_norm_stderr": 0.02725720260611495 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.19245283018867926, + "acc_stderr": 0.024262979839372277, + "acc_norm": 0.2830188679245283, + "acc_norm_stderr": 0.027724236492700904 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 6.251302083333333, + "likelihood_difference_stderr": 0.8500284067467659, + "pct_stereotype": 0.5416666666666666, + "pct_stereotype_stderr": 0.05913268547421811 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.84271978021978, + "likelihood_difference_stderr": 0.6726241671396568, + "pct_stereotype": 0.6593406593406593, + "pct_stereotype_stderr": 0.049956709512768704 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.24074074074074073, + "acc_stderr": 0.026067159222275794, + "acc_norm": 0.2814814814814815, + "acc_norm_stderr": 0.027420019350945263 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.18627450980392157, + "acc_stderr": 0.027325470966716326, + "acc_norm": 0.2696078431372549, + "acc_norm_stderr": 0.031145570659486782 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2765957446808511, + "acc_stderr": 0.026684564340460997, + "acc_norm": 0.2624113475177305, + "acc_norm_stderr": 0.026244920349843007 + }, + "hendrycksTest-computer_security": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.19327731092436976, + "acc_stderr": 0.02564947026588919, + "acc_norm": 0.2815126050420168, + "acc_norm_stderr": 0.029213549414372163 + }, + "hendrycksTest-human_aging": { + "acc": 0.2825112107623318, + "acc_stderr": 0.03021683101150878, + "acc_norm": 0.273542600896861, + "acc_norm_stderr": 0.029918586707798817 + }, + "crows_pairs_french": { + "likelihood_difference": 10.009703898330352, + "likelihood_difference_stderr": 0.2345615645708032, + "pct_stereotype": 0.5700655933214073, + "pct_stereotype_stderr": 0.01209278993435711 + }, + "hendrycksTest-prehistory": { + "acc": 0.23148148148148148, + "acc_stderr": 0.02346842983245115, + "acc_norm": 0.20679012345679013, + "acc_norm_stderr": 0.022535006705942818 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 7.706775700934579, + "likelihood_difference_stderr": 0.4545837073525256, + "pct_stereotype": 0.5327102803738317, + "pct_stereotype_stderr": 0.027890972865217984 + }, + "hendrycksTest-formal_logic": { + "acc": 0.24603174603174602, + "acc_stderr": 0.03852273364924318, + "acc_norm": 0.23809523809523808, + "acc_norm_stderr": 0.03809523809523809 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.21794871794871795, + "acc_stderr": 0.020932445774463196, + "acc_norm": 0.24358974358974358, + "acc_norm_stderr": 0.02176373368417392 + }, + "hendrycksTest-college_biology": { + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "acc_norm": 0.2638888888888889, + "acc_norm_stderr": 0.03685651095897532 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.17, + "acc_stderr": 0.0377525168068637, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036623 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 10.080780632411066, + "likelihood_difference_stderr": 0.5570562133400927, + "pct_stereotype": 0.5652173913043478, + "pct_stereotype_stderr": 0.03122795678881643 + }, + "hendrycksTest-virology": { + "acc": 0.1927710843373494, + "acc_stderr": 0.030709824050565274, + "acc_norm": 0.22289156626506024, + "acc_norm_stderr": 0.03240004825594688 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + } + }, + "versions": { + "crows_pairs_english_disability": 0, + "crows_pairs_french_sexual_orientation": 0, + "logiqa": 0, + "sciq": 0, + "arc_easy": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-philosophy": 0, + "crows_pairs_english_sexual_orientation": 0, + "crows_pairs_english": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-conceptual_physics": 0, + "arc_challenge": 0, + "crows_pairs_french_religion": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-global_facts": 0, + "lambada_openai": 0, + "winogrande": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-jurisprudence": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-college_mathematics": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-management": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-high_school_geography": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-high_school_statistics": 0, + "crows_pairs_french_disability": 0, + "wsc": 0, + "hendrycksTest-moral_disputes": 0, + "piqa": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-professional_psychology": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-econometrics": 0, + "crows_pairs_french_physical_appearance": 0, + "crows_pairs_french_age": 0, + "crows_pairs_french_race_color": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_english_physical_appearance": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-human_aging": 0, + "crows_pairs_french": 0, + "hendrycksTest-prehistory": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-college_chemistry": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-medical_genetics": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step2", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step23000.json b/data/checkpoint_eval/eleutherai_evals/step23000.json new file mode 100644 index 0000000000000000000000000000000000000000..89e0b024420ec07dbdbb2d5b4d47bdcb1df379f3 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step23000.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-logical_fallacies": { + "acc": 0.20245398773006135, + "acc_stderr": 0.031570650789119005, + "acc_norm": 0.3312883435582822, + "acc_norm_stderr": 0.03697983910025588 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.24, + "acc_stderr": 0.04292346959909284, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.623579545454546, + "likelihood_difference_stderr": 2.0995916372617076, + "pct_stereotype": 0.5454545454545454, + "pct_stereotype_stderr": 0.1574591643244434 + }, + "hendrycksTest-anatomy": { + "acc": 0.17777777777777778, + "acc_stderr": 0.03302789859901717, + "acc_norm": 0.1925925925925926, + "acc_norm_stderr": 0.0340654205850265 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.588541666666667, + "likelihood_difference_stderr": 0.6318561818523015, + "pct_stereotype": 0.3939393939393939, + "pct_stereotype_stderr": 0.06060606060606063 + }, + "lambada_openai": { + "ppl": 121.29880202709046, + "ppl_stderr": 5.126206628890121, + "acc": 0.2225887832330681, + "acc_stderr": 0.005795476001421499 + }, + "hendrycksTest-prehistory": { + "acc": 0.27469135802469136, + "acc_stderr": 0.024836057868294677, + "acc_norm": 0.20679012345679013, + "acc_norm_stderr": 0.02253500670594282 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.26, + "acc_norm_stderr": 0.04408440022768079 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.23404255319148937, + "acc_stderr": 0.027678452578212404, + "acc_norm": 0.20425531914893616, + "acc_norm_stderr": 0.026355158413349424 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.7319878472222223, + "likelihood_difference_stderr": 0.367552757457845, + "pct_stereotype": 0.5833333333333334, + "pct_stereotype_stderr": 0.05850912479161746 + }, + "hendrycksTest-virology": { + "acc": 0.25301204819277107, + "acc_stderr": 0.033844291552331346, + "acc_norm": 0.2289156626506024, + "acc_norm_stderr": 0.03270745277352477 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.2251655629139073, + "acc_stderr": 0.03410435282008937, + "acc_norm": 0.2251655629139073, + "acc_norm_stderr": 0.03410435282008937 + }, + "piqa": { + "acc": 0.6077257889009793, + "acc_stderr": 0.01139184674407223, + "acc_norm": 0.5968443960826986, + "acc_norm_stderr": 0.011444908701768742 + }, + "hendrycksTest-college_physics": { + "acc": 0.18627450980392157, + "acc_stderr": 0.03873958714149352, + "acc_norm": 0.23529411764705882, + "acc_norm_stderr": 0.04220773659171452 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.25925925925925924, + "acc_stderr": 0.04236511258094632, + "acc_norm": 0.37962962962962965, + "acc_norm_stderr": 0.04691521224077742 + }, + "hendrycksTest-management": { + "acc": 0.23300970873786409, + "acc_stderr": 0.04185832598928313, + "acc_norm": 0.2621359223300971, + "acc_norm_stderr": 0.04354631077260597 + }, + "hendrycksTest-machine_learning": { + "acc": 0.30357142857142855, + "acc_stderr": 0.04364226155841044, + "acc_norm": 0.1875, + "acc_norm_stderr": 0.0370468111477387 + }, + "hendrycksTest-public_relations": { + "acc": 0.3181818181818182, + "acc_stderr": 0.04461272175910508, + "acc_norm": 0.17272727272727273, + "acc_norm_stderr": 0.03620691833929219 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.5847077546296298, + "likelihood_difference_stderr": 0.27013774973583443, + "pct_stereotype": 0.4212962962962963, + "pct_stereotype_stderr": 0.03367462138896078 + }, + "crows_pairs_french": { + "likelihood_difference": 5.100849731663685, + "likelihood_difference_stderr": 0.1250618981891961, + "pct_stereotype": 0.4561717352415027, + "pct_stereotype_stderr": 0.012166287275376289 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.24352331606217617, + "acc_stderr": 0.030975436386845436, + "acc_norm": 0.24352331606217617, + "acc_norm_stderr": 0.03097543638684543 + }, + "hendrycksTest-sociology": { + "acc": 0.3034825870646766, + "acc_stderr": 0.03251006816458619, + "acc_norm": 0.2885572139303483, + "acc_norm_stderr": 0.03203841040213321 + }, + "hendrycksTest-international_law": { + "acc": 0.19008264462809918, + "acc_stderr": 0.03581796951709282, + "acc_norm": 0.4297520661157025, + "acc_norm_stderr": 0.04519082021319774 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.17037037037037037, + "acc_stderr": 0.022922554863074956, + "acc_norm": 0.24814814814814815, + "acc_norm_stderr": 0.0263357394040558 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.7034684065934065, + "likelihood_difference_stderr": 0.24752294883796136, + "pct_stereotype": 0.5054945054945055, + "pct_stereotype_stderr": 0.05270144531112881 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23687150837988827, + "acc_stderr": 0.01421957078810399, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "hendrycksTest-marketing": { + "acc": 0.2606837606837607, + "acc_stderr": 0.02876034895652341, + "acc_norm": 0.27350427350427353, + "acc_norm_stderr": 0.02920254015343117 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.1712962962962963, + "acc_stderr": 0.025695341643824688, + "acc_norm": 0.25462962962962965, + "acc_norm_stderr": 0.029711275860005344 + }, + "logiqa": { + "acc": 0.20583717357910905, + "acc_stderr": 0.01585842321932388, + "acc_norm": 0.30568356374807987, + "acc_norm_stderr": 0.018069997343763473 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.22058823529411764, + "acc_stderr": 0.029102254389674093, + "acc_norm": 0.24509803921568626, + "acc_norm_stderr": 0.03019028245350194 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.896134868421053, + "likelihood_difference_stderr": 0.2740385405281198, + "pct_stereotype": 0.631578947368421, + "pct_stereotype_stderr": 0.03508771929824559 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.24242424242424243, + "acc_stderr": 0.03346409881055953, + "acc_norm": 0.296969696969697, + "acc_norm_stderr": 0.03567969772268048 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.23225806451612904, + "acc_stderr": 0.024022256130308235, + "acc_norm": 0.3, + "acc_norm_stderr": 0.026069362295335134 + }, + "hendrycksTest-world_religions": { + "acc": 0.26900584795321636, + "acc_stderr": 0.03401052620104089, + "acc_norm": 0.3216374269005848, + "acc_norm_stderr": 0.03582529442573122 + }, + "hendrycksTest-college_medicine": { + "acc": 0.24855491329479767, + "acc_stderr": 0.03295304696818317, + "acc_norm": 0.31213872832369943, + "acc_norm_stderr": 0.03533133389323657 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.26262626262626265, + "acc_stderr": 0.031353050095330855, + "acc_norm": 0.2727272727272727, + "acc_norm_stderr": 0.03173071239071724 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206824 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.371294466403162, + "likelihood_difference_stderr": 0.39282681114851664, + "pct_stereotype": 0.2845849802371542, + "pct_stereotype_stderr": 0.02842397052208522 + }, + "hendrycksTest-philosophy": { + "acc": 0.19935691318327975, + "acc_stderr": 0.022691033780549656, + "acc_norm": 0.2604501607717042, + "acc_norm_stderr": 0.02492672322484555 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.2132472826086955, + "likelihood_difference_stderr": 0.2256584973219656, + "pct_stereotype": 0.3673913043478261, + "pct_stereotype_stderr": 0.022502235852959178 + }, + "hendrycksTest-computer_security": { + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "acc_norm": 0.27, + "acc_norm_stderr": 0.044619604333847394 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2900763358778626, + "acc_stderr": 0.03980066246467765, + "acc_norm": 0.2595419847328244, + "acc_norm_stderr": 0.03844876139785271 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.20588235294117646, + "acc_stderr": 0.026265024608275882, + "acc_norm": 0.31932773109243695, + "acc_norm_stderr": 0.030283995525884396 + }, + "hendrycksTest-human_aging": { + "acc": 0.25112107623318386, + "acc_stderr": 0.029105220833224598, + "acc_norm": 0.23318385650224216, + "acc_norm_stderr": 0.028380391147094713 + }, + "hendrycksTest-astronomy": { + "acc": 0.19736842105263158, + "acc_stderr": 0.03238981601699397, + "acc_norm": 0.3026315789473684, + "acc_norm_stderr": 0.037385206761196686 + }, + "sciq": { + "acc": 0.653, + "acc_stderr": 0.015060472031706615, + "acc_norm": 0.561, + "acc_norm_stderr": 0.015701131345400778 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.24102564102564103, + "acc_stderr": 0.02168554666533318, + "acc_norm": 0.26666666666666666, + "acc_norm_stderr": 0.022421273612923717 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.29044117647058826, + "acc_stderr": 0.027576468622740522, + "acc_norm": 0.27205882352941174, + "acc_norm_stderr": 0.027033041151681456 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.296551724137931, + "acc_stderr": 0.03806142687309994, + "acc_norm": 0.31724137931034485, + "acc_norm_stderr": 0.03878352372138622 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2624113475177305, + "acc_stderr": 0.026244920349842993, + "acc_norm": 0.23404255319148937, + "acc_norm_stderr": 0.025257861359432414 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.4817913385826773, + "likelihood_difference_stderr": 0.1785277431576976, + "pct_stereotype": 0.4822834645669291, + "pct_stereotype_stderr": 0.022191835500120254 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 5.0161401098901095, + "likelihood_difference_stderr": 0.46342554021316945, + "pct_stereotype": 0.7912087912087912, + "pct_stereotype_stderr": 0.04284305206509431 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.23202614379084968, + "acc_stderr": 0.017077373377856996, + "acc_norm": 0.2696078431372549, + "acc_norm_stderr": 0.017952449196987862 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.17, + "acc_stderr": 0.03775251680686371, + "acc_norm": 0.26, + "acc_norm_stderr": 0.0440844002276808 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.25871559633027524, + "acc_stderr": 0.01877605231961962, + "acc_norm": 0.25137614678899084, + "acc_norm_stderr": 0.01859920636028741 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.19052734375, + "likelihood_difference_stderr": 0.27369024853254786, + "pct_stereotype": 0.565625, + "pct_stereotype_stderr": 0.02775245248136476 + }, + "crows_pairs_french_age": { + "likelihood_difference": 5.053125, + "likelihood_difference_stderr": 0.49175013822402935, + "pct_stereotype": 0.4444444444444444, + "pct_stereotype_stderr": 0.05267171812666418 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 3.076923076923077, + "likelihood_difference_stderr": 0.5135827293235896, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.14390989949130545 + }, + "hendrycksTest-business_ethics": { + "acc": 0.35, + "acc_stderr": 0.04793724854411019, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.696572580645161, + "likelihood_difference_stderr": 0.5990114491774148, + "pct_stereotype": 0.7634408602150538, + "pct_stereotype_stderr": 0.04430611317732682 + }, + "hendrycksTest-nutrition": { + "acc": 0.27450980392156865, + "acc_stderr": 0.025553169991826514, + "acc_norm": 0.35947712418300654, + "acc_norm_stderr": 0.027475969910660952 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.24393358876117496, + "acc_stderr": 0.015357212665829496, + "acc_norm": 0.2413793103448276, + "acc_norm_stderr": 0.015302380123542094 + }, + "crows_pairs_english": { + "likelihood_difference": 3.5903957960644006, + "likelihood_difference_stderr": 0.10456293216351101, + "pct_stereotype": 0.5426356589147286, + "pct_stereotype_stderr": 0.01216881555248585 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.17733990147783252, + "acc_stderr": 0.02687433727680835, + "acc_norm": 0.26108374384236455, + "acc_norm_stderr": 0.03090379695211447 + }, + "hendrycksTest-formal_logic": { + "acc": 0.2857142857142857, + "acc_stderr": 0.04040610178208841, + "acc_norm": 0.24603174603174602, + "acc_norm_stderr": 0.03852273364924315 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206845 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 4.474330357142857, + "likelihood_difference_stderr": 0.36811002483302646, + "pct_stereotype": 0.4642857142857143, + "pct_stereotype_stderr": 0.035714285714285705 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 4.437771739130435, + "likelihood_difference_stderr": 0.5024669037031032, + "pct_stereotype": 0.6, + "pct_stereotype_stderr": 0.04588314677411234 + }, + "hendrycksTest-college_biology": { + "acc": 0.2916666666666667, + "acc_stderr": 0.038009680605548574, + "acc_norm": 0.24305555555555555, + "acc_norm_stderr": 0.0358687928008034 + }, + "hendrycksTest-econometrics": { + "acc": 0.22807017543859648, + "acc_stderr": 0.03947152782669415, + "acc_norm": 0.2631578947368421, + "acc_norm_stderr": 0.04142439719489362 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.053819444444445, + "likelihood_difference_stderr": 0.6134387786113794, + "pct_stereotype": 0.5555555555555556, + "pct_stereotype_stderr": 0.05897165471491952 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.26011560693641617, + "acc_stderr": 0.023618678310069363, + "acc_norm": 0.2832369942196532, + "acc_norm_stderr": 0.024257901705323374 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 5.027453271028038, + "likelihood_difference_stderr": 0.2316347109876809, + "pct_stereotype": 0.557632398753894, + "pct_stereotype_stderr": 0.02776455173721248 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.24050632911392406, + "acc_stderr": 0.027820781981149675, + "acc_norm": 0.3037974683544304, + "acc_norm_stderr": 0.02993669638713861 + }, + "arc_challenge": { + "acc": 0.18600682593856654, + "acc_stderr": 0.011370940183266759, + "acc_norm": 0.20477815699658702, + "acc_norm_stderr": 0.011792544338513405 + }, + "winogrande": { + "acc": 0.48539857932123126, + "acc_stderr": 0.014046492383275846 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.318130630630631, + "likelihood_difference_stderr": 0.4059063464506704, + "pct_stereotype": 0.6036036036036037, + "pct_stereotype_stderr": 0.04663848326322448 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "acc_norm": 0.27, + "acc_norm_stderr": 0.0446196043338474 + }, + "hendrycksTest-global_facts": { + "acc": 0.22, + "acc_stderr": 0.041633319989322674, + "acc_norm": 0.24, + "acc_norm_stderr": 0.042923469599092816 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.24338624338624337, + "acc_stderr": 0.022101128787415436, + "acc_norm": 0.2619047619047619, + "acc_norm_stderr": 0.022644212615525218 + }, + "hendrycksTest-security_studies": { + "acc": 0.3020408163265306, + "acc_stderr": 0.029393609319879818, + "acc_norm": 0.23673469387755103, + "acc_norm_stderr": 0.027212835884073163 + }, + "hendrycksTest-professional_law": { + "acc": 0.26140808344198174, + "acc_stderr": 0.011222528169771312, + "acc_norm": 0.2790091264667536, + "acc_norm_stderr": 0.011455208832803534 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.156009615384615, + "likelihood_difference_stderr": 0.6114325636078373, + "pct_stereotype": 0.6307692307692307, + "pct_stereotype_stderr": 0.060324565928300454 + }, + "arc_easy": { + "acc": 0.3952020202020202, + "acc_stderr": 0.01003189405279098, + "acc_norm": 0.3602693602693603, + "acc_norm_stderr": 0.009851002584732387 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.21, + "acc_stderr": 0.04093601807403326, + "acc_norm": 0.35, + "acc_norm_stderr": 0.0479372485441102 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.24150943396226415, + "acc_stderr": 0.026341480371118362, + "acc_norm": 0.3132075471698113, + "acc_norm_stderr": 0.02854479331905533 + } + }, + "versions": { + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-high_school_computer_science": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-anatomy": 0, + "crows_pairs_french_disability": 0, + "lambada_openai": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-conceptual_physics": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-high_school_physics": 0, + "piqa": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-management": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-public_relations": 0, + "wsc": 0, + "crows_pairs_english_nationality": 0, + "crows_pairs_french": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-high_school_mathematics": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-high_school_statistics": 0, + "logiqa": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-abstract_algebra": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-philosophy": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-astronomy": 0, + "sciq": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-professional_accounting": 0, + "crows_pairs_english_race_color": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-high_school_psychology": 0, + "crows_pairs_english_gender": 0, + "crows_pairs_french_age": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-business_ethics": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-miscellaneous": 0, + "crows_pairs_english": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-us_foreign_policy": 0, + "crows_pairs_french_socioeconomic": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-econometrics": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-moral_disputes": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-high_school_world_history": 0, + "arc_challenge": 0, + "winogrande": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-professional_law": 0, + "crows_pairs_english_disability": 0, + "arc_easy": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-clinical_knowledge": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step23000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:2", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step256.json b/data/checkpoint_eval/eleutherai_evals/step256.json new file mode 100644 index 0000000000000000000000000000000000000000..4d7426427b50909b67efe4e72ff0f38a877b76f3 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step256.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_french_gender": { + "likelihood_difference": 5.687986760124611, + "likelihood_difference_stderr": 0.2761406584883121, + "pct_stereotype": 0.470404984423676, + "pct_stereotype_stderr": 0.02790184442005117 + }, + "hendrycksTest-marketing": { + "acc": 0.23076923076923078, + "acc_stderr": 0.027601921381417604, + "acc_norm": 0.23076923076923078, + "acc_norm_stderr": 0.027601921381417604 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.24587155963302754, + "acc_stderr": 0.01846194096870845, + "acc_norm": 0.26972477064220185, + "acc_norm_stderr": 0.019028486711115445 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.23, + "acc_stderr": 0.04229525846816507, + "acc_norm": 0.23, + "acc_norm_stderr": 0.042295258468165065 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.14, + "acc_norm_stderr": 0.03487350880197772 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.1921182266009852, + "acc_stderr": 0.02771931570961477, + "acc_norm": 0.22167487684729065, + "acc_norm_stderr": 0.029225575892489614 + }, + "hendrycksTest-econometrics": { + "acc": 0.2543859649122807, + "acc_stderr": 0.040969851398436695, + "acc_norm": 0.2719298245614035, + "acc_norm_stderr": 0.04185774424022056 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 6.718269230769231, + "likelihood_difference_stderr": 0.8596632745046646, + "pct_stereotype": 0.4461538461538462, + "pct_stereotype_stderr": 0.06213651700539812 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "acc_norm": 0.29, + "acc_norm_stderr": 0.04560480215720684 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 6.856966403162056, + "likelihood_difference_stderr": 0.32916109908316876, + "pct_stereotype": 0.5454545454545454, + "pct_stereotype_stderr": 0.0313666163337434 + }, + "hendrycksTest-public_relations": { + "acc": 0.2818181818181818, + "acc_stderr": 0.04309118709946458, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.036942843353377997 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.380837912087912, + "likelihood_difference_stderr": 0.49983969692767516, + "pct_stereotype": 0.5164835164835165, + "pct_stereotype_stderr": 0.05267597952306975 + }, + "logiqa": { + "acc": 0.21044546850998463, + "acc_stderr": 0.015988369488888755, + "acc_norm": 0.23348694316436253, + "acc_norm_stderr": 0.016593362460570887 + }, + "hendrycksTest-human_aging": { + "acc": 0.3273542600896861, + "acc_stderr": 0.03149384670994131, + "acc_norm": 0.29596412556053814, + "acc_norm_stderr": 0.03063659134869981 + }, + "hendrycksTest-formal_logic": { + "acc": 0.2698412698412698, + "acc_stderr": 0.03970158273235172, + "acc_norm": 0.30158730158730157, + "acc_norm_stderr": 0.04104947269903394 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.18490566037735848, + "acc_stderr": 0.023893351834464324, + "acc_norm": 0.3169811320754717, + "acc_norm_stderr": 0.02863723563980091 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 5.5, + "likelihood_difference_stderr": 0.9504975080559196, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.14390989949130545 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.2254335260115607, + "acc_stderr": 0.02249723019096755, + "acc_norm": 0.22832369942196531, + "acc_norm_stderr": 0.022598703804321624 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.677083333333333, + "likelihood_difference_stderr": 0.6194240763408452, + "pct_stereotype": 0.6451612903225806, + "pct_stereotype_stderr": 0.049883363937668256 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.25326797385620914, + "acc_stderr": 0.01759348689536683, + "acc_norm": 0.272875816993464, + "acc_norm_stderr": 0.018020474148393577 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.18907563025210083, + "acc_stderr": 0.025435119438105357, + "acc_norm": 0.2773109243697479, + "acc_norm_stderr": 0.02907937453948001 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.2175925925925926, + "acc_stderr": 0.028139689444859645, + "acc_norm": 0.23148148148148148, + "acc_norm_stderr": 0.028765111718046944 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 4.05078125, + "likelihood_difference_stderr": 0.38152405690444796, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.027994625547792713 + }, + "wsc": { + "acc": 0.6346153846153846, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.18627450980392157, + "acc_stderr": 0.027325470966716336, + "acc_norm": 0.21568627450980393, + "acc_norm_stderr": 0.028867431449849313 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 5.009853603603603, + "likelihood_difference_stderr": 0.5228133914951523, + "pct_stereotype": 0.5855855855855856, + "pct_stereotype_stderr": 0.04696953631102271 + }, + "sciq": { + "acc": 0.228, + "acc_stderr": 0.013273740700804483, + "acc_norm": 0.236, + "acc_norm_stderr": 0.013434451402438685 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 4.72265625, + "likelihood_difference_stderr": 0.5793499299137083, + "pct_stereotype": 0.5555555555555556, + "pct_stereotype_stderr": 0.05897165471491952 + }, + "hendrycksTest-machine_learning": { + "acc": 0.2767857142857143, + "acc_stderr": 0.042466243366976256, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04109974682633932 + }, + "hendrycksTest-prehistory": { + "acc": 0.28703703703703703, + "acc_stderr": 0.02517104191530968, + "acc_norm": 0.24382716049382716, + "acc_norm_stderr": 0.023891879541959593 + }, + "hendrycksTest-sociology": { + "acc": 0.23383084577114427, + "acc_stderr": 0.029929415408348384, + "acc_norm": 0.24875621890547264, + "acc_norm_stderr": 0.03056767593891672 + }, + "hendrycksTest-global_facts": { + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "crows_pairs_french": { + "likelihood_difference": 6.74689736135957, + "likelihood_difference_stderr": 0.15103608824599826, + "pct_stereotype": 0.5533691115086464, + "pct_stereotype_stderr": 0.012143526564900555 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "acc_norm": 0.28, + "acc_norm_stderr": 0.045126085985421276 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.12222222222222222, + "acc_stderr": 0.019970605780284603, + "acc_norm": 0.1814814814814815, + "acc_norm_stderr": 0.023499264669407282 + }, + "hendrycksTest-college_biology": { + "acc": 0.2222222222222222, + "acc_stderr": 0.034765901043041336, + "acc_norm": 0.20833333333333334, + "acc_norm_stderr": 0.03396116205845335 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.251063829787234, + "acc_stderr": 0.02834696377716246, + "acc_norm": 0.2, + "acc_norm_stderr": 0.0261488180184245 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.1574074074074074, + "acc_stderr": 0.03520703990517965, + "acc_norm": 0.21296296296296297, + "acc_norm_stderr": 0.039578354719809805 + }, + "crows_pairs_english": { + "likelihood_difference": 4.661393112701252, + "likelihood_difference_stderr": 0.13998586074905606, + "pct_stereotype": 0.456768038163387, + "pct_stereotype_stderr": 0.012167560197793078 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 13.163461538461538, + "likelihood_difference_stderr": 0.8325716351947234, + "pct_stereotype": 0.7912087912087912, + "pct_stereotype_stderr": 0.042843052065094304 + }, + "hendrycksTest-management": { + "acc": 0.23300970873786409, + "acc_stderr": 0.04185832598928315, + "acc_norm": 0.2815533980582524, + "acc_norm_stderr": 0.04453254836326467 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 4.904440789473684, + "likelihood_difference_stderr": 0.4062917141669697, + "pct_stereotype": 0.48947368421052634, + "pct_stereotype_stderr": 0.036361587723547695 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.22085889570552147, + "acc_stderr": 0.032591773927421776, + "acc_norm": 0.3128834355828221, + "acc_norm_stderr": 0.036429145782924055 + }, + "hendrycksTest-astronomy": { + "acc": 0.20394736842105263, + "acc_stderr": 0.032790004063100495, + "acc_norm": 0.27631578947368424, + "acc_norm_stderr": 0.03639057569952925 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 6.349431818181818, + "likelihood_difference_stderr": 2.804745680840638, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.17721518987341772, + "acc_stderr": 0.02485636418450322, + "acc_norm": 0.25738396624472576, + "acc_norm_stderr": 0.028458820991460295 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.25, + "acc_stderr": 0.026303648393696036, + "acc_norm": 0.25, + "acc_norm_stderr": 0.026303648393696036 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.21, + "acc_norm_stderr": 0.040936018074033256 + }, + "lambada_openai": { + "ppl": 705314.6370389248, + "ppl_stderr": 50610.68705557734, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "hendrycksTest-college_medicine": { + "acc": 0.23699421965317918, + "acc_stderr": 0.03242414757483098, + "acc_norm": 0.2658959537572254, + "acc_norm_stderr": 0.03368762932259432 + }, + "arc_easy": { + "acc": 0.27441077441077444, + "acc_stderr": 0.00915617712224453, + "acc_norm": 0.2849326599326599, + "acc_norm_stderr": 0.009262170695590658 + }, + "hendrycksTest-security_studies": { + "acc": 0.3306122448979592, + "acc_stderr": 0.030116426296540613, + "acc_norm": 0.20408163265306123, + "acc_norm_stderr": 0.025801283475090506 + }, + "winogrande": { + "acc": 0.4925019731649566, + "acc_stderr": 0.014050905521228577 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 5.4428530092592595, + "likelihood_difference_stderr": 0.3840752204417463, + "pct_stereotype": 0.3333333333333333, + "pct_stereotype_stderr": 0.03214952147802749 + }, + "arc_challenge": { + "acc": 0.19965870307167236, + "acc_stderr": 0.011681625756888669, + "acc_norm": 0.24146757679180889, + "acc_norm_stderr": 0.01250656483973943 + }, + "hendrycksTest-computer_security": { + "acc": 0.22, + "acc_stderr": 0.04163331998932268, + "acc_norm": 0.27, + "acc_norm_stderr": 0.044619604333847394 + }, + "hendrycksTest-world_religions": { + "acc": 0.1695906432748538, + "acc_stderr": 0.028782108105401712, + "acc_norm": 0.25146198830409355, + "acc_norm_stderr": 0.033275044238468436 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.167361111111111, + "likelihood_difference_stderr": 0.49130810000225555, + "pct_stereotype": 0.4111111111111111, + "pct_stereotype_stderr": 0.052155640611075534 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.2275132275132275, + "acc_stderr": 0.021591269407823778, + "acc_norm": 0.21164021164021163, + "acc_norm_stderr": 0.02103733150526289 + }, + "hendrycksTest-international_law": { + "acc": 0.10743801652892562, + "acc_stderr": 0.02826881219254063, + "acc_norm": 0.2396694214876033, + "acc_norm_stderr": 0.03896878985070417 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 10.162878787878787, + "likelihood_difference_stderr": 1.04556369991972, + "pct_stereotype": 0.3333333333333333, + "pct_stereotype_stderr": 0.0584705346204686 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.23627075351213284, + "acc_stderr": 0.015190473717037498, + "acc_norm": 0.25287356321839083, + "acc_norm_stderr": 0.015543377313719681 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.16363636363636364, + "acc_stderr": 0.028887872395487953, + "acc_norm": 0.24242424242424243, + "acc_norm_stderr": 0.03346409881055953 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 7.765760869565217, + "likelihood_difference_stderr": 0.49195584086877725, + "pct_stereotype": 0.6869565217391305, + "pct_stereotype_stderr": 0.043432470166108225 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.25886524822695034, + "acc_stderr": 0.026129572527180848, + "acc_norm": 0.2730496453900709, + "acc_norm_stderr": 0.02657786094330786 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.18686868686868688, + "acc_stderr": 0.027772533334218977, + "acc_norm": 0.30303030303030304, + "acc_norm_stderr": 0.032742879140268674 + }, + "hendrycksTest-anatomy": { + "acc": 0.2074074074074074, + "acc_stderr": 0.03502553170678319, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.03785714465066653 + }, + "hendrycksTest-philosophy": { + "acc": 0.2379421221864952, + "acc_stderr": 0.02418515064781871, + "acc_norm": 0.2990353697749196, + "acc_norm_stderr": 0.02600330111788513 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 4.281742125984252, + "likelihood_difference_stderr": 0.21780058915583433, + "pct_stereotype": 0.3838582677165354, + "pct_stereotype_stderr": 0.021598410071068296 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.19689119170984457, + "acc_stderr": 0.028697873971860674, + "acc_norm": 0.2538860103626943, + "acc_norm_stderr": 0.03141024780565318 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.2052980132450331, + "acc_stderr": 0.03297986648473836, + "acc_norm": 0.24503311258278146, + "acc_norm_stderr": 0.035118075718047245 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 7.983976403061225, + "likelihood_difference_stderr": 0.545579868210259, + "pct_stereotype": 0.34183673469387754, + "pct_stereotype_stderr": 0.033967132039868675 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.19743589743589743, + "acc_stderr": 0.02018264696867484, + "acc_norm": 0.22564102564102564, + "acc_norm_stderr": 0.02119363252514852 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.29770992366412213, + "acc_stderr": 0.04010358942462203, + "acc_norm": 0.2824427480916031, + "acc_norm_stderr": 0.03948406125768361 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.25517241379310346, + "acc_stderr": 0.03632984052707842, + "acc_norm": 0.2689655172413793, + "acc_norm_stderr": 0.036951833116502325 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.24, + "acc_norm_stderr": 0.042923469599092816 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 5.3552989130434785, + "likelihood_difference_stderr": 0.2271004698936648, + "pct_stereotype": 0.6869565217391305, + "pct_stereotype_stderr": 0.021645150653106047 + }, + "piqa": { + "acc": 0.5179542981501633, + "acc_stderr": 0.011658300623287153, + "acc_norm": 0.515778019586507, + "acc_norm_stderr": 0.011660014400426182 + }, + "hendrycksTest-virology": { + "acc": 0.22289156626506024, + "acc_stderr": 0.03240004825594688, + "acc_norm": 0.25301204819277107, + "acc_norm_stderr": 0.03384429155233137 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.16, + "acc_stderr": 0.03684529491774708, + "acc_norm": 0.17, + "acc_norm_stderr": 0.0377525168068637 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.2129032258064516, + "acc_stderr": 0.02328766512726853, + "acc_norm": 0.23870967741935484, + "acc_norm_stderr": 0.024251071262208837 + }, + "hendrycksTest-professional_law": { + "acc": 0.242503259452412, + "acc_stderr": 0.010946570966348783, + "acc_norm": 0.2711864406779661, + "acc_norm_stderr": 0.011354581451622986 + }, + "hendrycksTest-college_physics": { + "acc": 0.20588235294117646, + "acc_stderr": 0.04023382273617747, + "acc_norm": 0.19607843137254902, + "acc_norm_stderr": 0.03950581861179962 + }, + "hendrycksTest-nutrition": { + "acc": 0.19607843137254902, + "acc_stderr": 0.022733789405447593, + "acc_norm": 0.28431372549019607, + "acc_norm_stderr": 0.025829163272757482 + }, + "hendrycksTest-business_ethics": { + "acc": 0.32, + "acc_stderr": 0.04688261722621505, + "acc_norm": 0.29, + "acc_norm_stderr": 0.04560480215720684 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 7.185329861111111, + "likelihood_difference_stderr": 0.9560662240150144, + "pct_stereotype": 0.5416666666666666, + "pct_stereotype_stderr": 0.05913268547421809 + } + }, + "versions": { + "crows_pairs_french_gender": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-econometrics": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-high_school_computer_science": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-public_relations": 0, + "crows_pairs_english_age": 0, + "logiqa": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-moral_disputes": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-high_school_statistics": 0, + "crows_pairs_english_gender": 0, + "wsc": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_english_religion": 0, + "sciq": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-global_facts": 0, + "crows_pairs_french": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-jurisprudence": 0, + "crows_pairs_english": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-management": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-astronomy": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-college_computer_science": 0, + "lambada_openai": 0, + "hendrycksTest-college_medicine": 0, + "arc_easy": 0, + "hendrycksTest-security_studies": 0, + "winogrande": 0, + "crows_pairs_english_nationality": 0, + "arc_challenge": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-international_law": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-high_school_european_history": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-philosophy": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-high_school_physics": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-us_foreign_policy": 0, + "crows_pairs_french_race_color": 0, + "piqa": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-business_ethics": 0, + "crows_pairs_french_physical_appearance": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step256", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step3000.json b/data/checkpoint_eval/eleutherai_evals/step3000.json new file mode 100644 index 0000000000000000000000000000000000000000..22f573e0c4111b4ffee01f6f78cf039ac6e1a869 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step3000.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_english": { + "likelihood_difference": 3.4712842874180083, + "likelihood_difference_stderr": 0.10356437803282284, + "pct_stereotype": 0.528324388789505, + "pct_stereotype_stderr": 0.012193686719906043 + }, + "hendrycksTest-international_law": { + "acc": 0.15702479338842976, + "acc_stderr": 0.0332124484254713, + "acc_norm": 0.3884297520661157, + "acc_norm_stderr": 0.04449270350068381 + }, + "hendrycksTest-sociology": { + "acc": 0.2736318407960199, + "acc_stderr": 0.03152439186555405, + "acc_norm": 0.29850746268656714, + "acc_norm_stderr": 0.032357437893550424 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-econometrics": { + "acc": 0.2631578947368421, + "acc_stderr": 0.0414243971948936, + "acc_norm": 0.2631578947368421, + "acc_norm_stderr": 0.0414243971948936 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.25517241379310346, + "acc_stderr": 0.03632984052707842, + "acc_norm": 0.2827586206896552, + "acc_norm_stderr": 0.03752833958003337 + }, + "hendrycksTest-business_ethics": { + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-virology": { + "acc": 0.2710843373493976, + "acc_stderr": 0.03460579907553028, + "acc_norm": 0.2289156626506024, + "acc_norm_stderr": 0.03270745277352477 + }, + "hendrycksTest-nutrition": { + "acc": 0.24836601307189543, + "acc_stderr": 0.024739981355113596, + "acc_norm": 0.32679738562091504, + "acc_norm_stderr": 0.026857294663281423 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 6.3964673913043475, + "likelihood_difference_stderr": 0.5088370601767548, + "pct_stereotype": 0.45217391304347826, + "pct_stereotype_stderr": 0.04661456979958347 + }, + "crows_pairs_french": { + "likelihood_difference": 5.3115170319022065, + "likelihood_difference_stderr": 0.13097278073096086, + "pct_stereotype": 0.4502087060226595, + "pct_stereotype_stderr": 0.012152590574174895 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.093165887850467, + "likelihood_difference_stderr": 0.22263346692021055, + "pct_stereotype": 0.5233644859813084, + "pct_stereotype_stderr": 0.027920316348204993 + }, + "hendrycksTest-security_studies": { + "acc": 0.2693877551020408, + "acc_stderr": 0.02840125202902294, + "acc_norm": 0.22040816326530613, + "acc_norm_stderr": 0.026537045312145277 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036623 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.4659288194444446, + "likelihood_difference_stderr": 0.3806466296043766, + "pct_stereotype": 0.5416666666666666, + "pct_stereotype_stderr": 0.05913268547421811 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.26838235294117646, + "acc_stderr": 0.02691748122437721, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.026799562024887674 + }, + "hendrycksTest-astronomy": { + "acc": 0.23026315789473684, + "acc_stderr": 0.03426059424403165, + "acc_norm": 0.32894736842105265, + "acc_norm_stderr": 0.03823428969926605 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.25252525252525254, + "acc_stderr": 0.030954055470365904, + "acc_norm": 0.25757575757575757, + "acc_norm_stderr": 0.031156269519646836 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.595516304347826, + "likelihood_difference_stderr": 0.23328673581474416, + "pct_stereotype": 0.44130434782608696, + "pct_stereotype_stderr": 0.023176636328300308 + }, + "hendrycksTest-machine_learning": { + "acc": 0.26785714285714285, + "acc_stderr": 0.04203277291467762, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04109974682633932 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.661739864864865, + "likelihood_difference_stderr": 0.4569099844348636, + "pct_stereotype": 0.6036036036036037, + "pct_stereotype_stderr": 0.04663848326322448 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.19858156028368795, + "acc_stderr": 0.02379830163794214, + "acc_norm": 0.20567375886524822, + "acc_norm_stderr": 0.024112138950471887 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.34, + "acc_norm_stderr": 0.04760952285695235 + }, + "piqa": { + "acc": 0.5794341675734495, + "acc_stderr": 0.011517665611282774, + "acc_norm": 0.5837867247007617, + "acc_norm_stderr": 0.011500864675166568 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.26053639846743293, + "acc_stderr": 0.01569600856380708, + "acc_norm": 0.25287356321839083, + "acc_norm_stderr": 0.015543377313719681 + }, + "sciq": { + "acc": 0.592, + "acc_stderr": 0.015549205052920676, + "acc_norm": 0.515, + "acc_norm_stderr": 0.015812179641814902 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.2434640522875817, + "acc_stderr": 0.017362473762146634, + "acc_norm": 0.25, + "acc_norm_stderr": 0.01751781884501444 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.6025028935185186, + "likelihood_difference_stderr": 0.28513005796161467, + "pct_stereotype": 0.4166666666666667, + "pct_stereotype_stderr": 0.03362277436608043 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.18, + "acc_stderr": 0.038612291966536955, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909282 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 2.84423828125, + "likelihood_difference_stderr": 0.2759970404950795, + "pct_stereotype": 0.58125, + "pct_stereotype_stderr": 0.027622536202702143 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.2339622641509434, + "acc_stderr": 0.02605529690115292, + "acc_norm": 0.27169811320754716, + "acc_norm_stderr": 0.027377706624670713 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.205288461538461, + "likelihood_difference_stderr": 0.5700502266857143, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.06231481440776789 + }, + "hendrycksTest-public_relations": { + "acc": 0.3090909090909091, + "acc_stderr": 0.044262946482000985, + "acc_norm": 0.23636363636363636, + "acc_norm_stderr": 0.040693063197213775 + }, + "arc_challenge": { + "acc": 0.17918088737201365, + "acc_stderr": 0.011207045216615674, + "acc_norm": 0.21245733788395904, + "acc_norm_stderr": 0.011953482906582952 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.5685096153846154, + "likelihood_difference_stderr": 0.2910998803105466, + "pct_stereotype": 0.4945054945054945, + "pct_stereotype_stderr": 0.052701445311128796 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.2777777777777778, + "acc_stderr": 0.02306818884826111, + "acc_norm": 0.2804232804232804, + "acc_norm_stderr": 0.023135287974325628 + }, + "crows_pairs_french_age": { + "likelihood_difference": 3.855208333333333, + "likelihood_difference_stderr": 0.4788440459459206, + "pct_stereotype": 0.45555555555555555, + "pct_stereotype_stderr": 0.05279009646630345 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.4081439393939394, + "likelihood_difference_stderr": 0.5962932736116068, + "pct_stereotype": 0.48484848484848486, + "pct_stereotype_stderr": 0.06198888629778894 + }, + "winogrande": { + "acc": 0.5082872928176796, + "acc_stderr": 0.014050555322824192 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "acc_norm": 0.29, + "acc_norm_stderr": 0.04560480215720683 + }, + "hendrycksTest-professional_law": { + "acc": 0.23728813559322035, + "acc_stderr": 0.010865436690780269, + "acc_norm": 0.2646675358539765, + "acc_norm_stderr": 0.011267332992845528 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.27, + "acc_stderr": 0.0446196043338474, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206845 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.9288651315789473, + "likelihood_difference_stderr": 0.2802051715338846, + "pct_stereotype": 0.6473684210526316, + "pct_stereotype_stderr": 0.03475405259582098 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.26717557251908397, + "acc_stderr": 0.03880848301082394, + "acc_norm": 0.25190839694656486, + "acc_norm_stderr": 0.03807387116306086 + }, + "arc_easy": { + "acc": 0.35058922558922556, + "acc_stderr": 0.00979100382983156, + "acc_norm": 0.3354377104377104, + "acc_norm_stderr": 0.009688175165829592 + }, + "hendrycksTest-college_physics": { + "acc": 0.19607843137254902, + "acc_stderr": 0.03950581861179963, + "acc_norm": 0.28431372549019607, + "acc_norm_stderr": 0.04488482852329017 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 5.685096153846154, + "likelihood_difference_stderr": 0.5726006934973705, + "pct_stereotype": 0.7802197802197802, + "pct_stereotype_stderr": 0.04364972632898534 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.18134715025906736, + "acc_stderr": 0.02780703236068609, + "acc_norm": 0.23834196891191708, + "acc_norm_stderr": 0.030748905363909895 + }, + "hendrycksTest-human_aging": { + "acc": 0.3094170403587444, + "acc_stderr": 0.03102441174057221, + "acc_norm": 0.27802690582959644, + "acc_norm_stderr": 0.030069584874494033 + }, + "hendrycksTest-formal_logic": { + "acc": 0.2698412698412698, + "acc_stderr": 0.039701582732351706, + "acc_norm": 0.21428571428571427, + "acc_norm_stderr": 0.03670066451047181 + }, + "logiqa": { + "acc": 0.22887864823348694, + "acc_stderr": 0.016478107276313284, + "acc_norm": 0.28110599078341014, + "acc_norm_stderr": 0.017632374626460005 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 6.457465277777778, + "likelihood_difference_stderr": 0.6653048237221467, + "pct_stereotype": 0.5138888888888888, + "pct_stereotype_stderr": 0.05931618532716555 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.21428571428571427, + "acc_stderr": 0.02665353159671548, + "acc_norm": 0.31932773109243695, + "acc_norm_stderr": 0.030283995525884396 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.25483870967741934, + "acc_stderr": 0.02479011845933221, + "acc_norm": 0.2806451612903226, + "acc_norm_stderr": 0.025560604721022895 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.22549019607843138, + "acc_stderr": 0.02933116229425172, + "acc_norm": 0.25980392156862747, + "acc_norm_stderr": 0.03077855467869327 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.2616033755274262, + "acc_stderr": 0.028609516716994934, + "acc_norm": 0.28270042194092826, + "acc_norm_stderr": 0.029312814153955924 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.26605504587155965, + "acc_stderr": 0.018946022322225604, + "acc_norm": 0.26788990825688075, + "acc_norm_stderr": 0.018987462257978652 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.23829787234042554, + "acc_stderr": 0.027851252973889778, + "acc_norm": 0.2127659574468085, + "acc_norm_stderr": 0.026754391348039766 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.15270935960591134, + "acc_stderr": 0.025308904539380644, + "acc_norm": 0.28078817733990147, + "acc_norm_stderr": 0.031618563353586086 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 8.12685276679842, + "likelihood_difference_stderr": 0.41139398422096635, + "pct_stereotype": 0.233201581027668, + "pct_stereotype_stderr": 0.026638273845497513 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.2230769230769231, + "acc_stderr": 0.021107730127243998, + "acc_norm": 0.24615384615384617, + "acc_norm_stderr": 0.02184086699042308 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.108198924731183, + "likelihood_difference_stderr": 0.5195853940706195, + "pct_stereotype": 0.7204301075268817, + "pct_stereotype_stderr": 0.046789371667506734 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.16296296296296298, + "acc_stderr": 0.022518561997682648, + "acc_norm": 0.24444444444444444, + "acc_norm_stderr": 0.02620276653465215 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 2.8365384615384617, + "likelihood_difference_stderr": 0.7093355864720875, + "pct_stereotype": 0.3076923076923077, + "pct_stereotype_stderr": 0.13323467750529824 + }, + "hendrycksTest-college_medicine": { + "acc": 0.26011560693641617, + "acc_stderr": 0.033450369167889904, + "acc_norm": 0.2774566473988439, + "acc_norm_stderr": 0.034140140070440354 + }, + "hendrycksTest-management": { + "acc": 0.21359223300970873, + "acc_stderr": 0.040580420156460344, + "acc_norm": 0.2524271844660194, + "acc_norm_stderr": 0.04301250399690877 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542127 + }, + "hendrycksTest-world_religions": { + "acc": 0.23976608187134502, + "acc_stderr": 0.03274485211946956, + "acc_norm": 0.30409356725146197, + "acc_norm_stderr": 0.03528211258245232 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.24692737430167597, + "acc_stderr": 0.014422292204808835, + "acc_norm": 0.25139664804469275, + "acc_norm_stderr": 0.014508979453553972 + }, + "hendrycksTest-global_facts": { + "acc": 0.19, + "acc_stderr": 0.03942772444036625, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816506 + }, + "hendrycksTest-computer_security": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.3907787893700787, + "likelihood_difference_stderr": 0.16817580159222853, + "pct_stereotype": 0.4507874015748031, + "pct_stereotype_stderr": 0.02209795835867595 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.21818181818181817, + "acc_stderr": 0.03225078108306289, + "acc_norm": 0.2909090909090909, + "acc_norm_stderr": 0.03546563019624336 + }, + "hendrycksTest-marketing": { + "acc": 0.2222222222222222, + "acc_stderr": 0.027236013946196708, + "acc_norm": 0.2606837606837607, + "acc_norm_stderr": 0.028760348956523414 + }, + "hendrycksTest-philosophy": { + "acc": 0.19935691318327975, + "acc_stderr": 0.022691033780549656, + "acc_norm": 0.2829581993569132, + "acc_norm_stderr": 0.025583062489984824 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.23178807947019867, + "acc_stderr": 0.03445406271987053, + "acc_norm": 0.23178807947019867, + "acc_norm_stderr": 0.03445406271987054 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 4.900568181818182, + "likelihood_difference_stderr": 1.7545892452142433, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "lambada_openai": { + "ppl": 411.658325603736, + "ppl_stderr": 17.894759386978997, + "acc": 0.12128856976518533, + "acc_stderr": 0.004548258586998434 + }, + "hendrycksTest-college_biology": { + "acc": 0.2847222222222222, + "acc_stderr": 0.037738099906869334, + "acc_norm": 0.2708333333333333, + "acc_norm_stderr": 0.037161774375660185 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.22699386503067484, + "acc_stderr": 0.03291099578615769, + "acc_norm": 0.26380368098159507, + "acc_norm_stderr": 0.03462419931615622 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.19, + "acc_stderr": 0.03942772444036623, + "acc_norm": 0.2, + "acc_norm_stderr": 0.040201512610368445 + }, + "hendrycksTest-anatomy": { + "acc": 0.28888888888888886, + "acc_stderr": 0.03915450630414251, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.03785714465066655 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.21098265895953758, + "acc_stderr": 0.021966309947043135, + "acc_norm": 0.3063583815028902, + "acc_norm_stderr": 0.02481835012943659 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.23148148148148148, + "acc_stderr": 0.04077494709252626, + "acc_norm": 0.35185185185185186, + "acc_norm_stderr": 0.04616631111801713 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.19907407407407407, + "acc_stderr": 0.027232298462690242, + "acc_norm": 0.2361111111111111, + "acc_norm_stderr": 0.028963702570791037 + }, + "hendrycksTest-prehistory": { + "acc": 0.26851851851851855, + "acc_stderr": 0.024659685185967273, + "acc_norm": 0.22839506172839505, + "acc_norm_stderr": 0.023358211840626267 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 4.585817920918367, + "likelihood_difference_stderr": 0.3843067957203524, + "pct_stereotype": 0.4489795918367347, + "pct_stereotype_stderr": 0.03561884533975954 + } + }, + "versions": { + "crows_pairs_english": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-sociology": 0, + "wsc": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-nutrition": 0, + "crows_pairs_french_religion": 0, + "crows_pairs_french": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-college_computer_science": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-high_school_geography": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-machine_learning": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-medical_genetics": 0, + "piqa": 0, + "hendrycksTest-miscellaneous": 0, + "sciq": 0, + "hendrycksTest-professional_psychology": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-college_mathematics": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-public_relations": 0, + "arc_challenge": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-elementary_mathematics": 0, + "crows_pairs_french_age": 0, + "crows_pairs_french_disability": 0, + "winogrande": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-college_chemistry": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-human_sexuality": 0, + "arc_easy": 0, + "hendrycksTest-college_physics": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-formal_logic": 0, + "logiqa": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-high_school_chemistry": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-high_school_mathematics": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-management": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-computer_security": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-high_school_physics": 0, + "crows_pairs_english_autre": 0, + "lambada_openai": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-prehistory": 0, + "crows_pairs_french_socioeconomic": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step3000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:0", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step32.json b/data/checkpoint_eval/eleutherai_evals/step32.json new file mode 100644 index 0000000000000000000000000000000000000000..18af7035e2489da78cf837b945e1179cd84dc88b --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step32.json @@ -0,0 +1,622 @@ +{ + "results": { + "lambada_openai": { + "ppl": 3288862.4386760374, + "ppl_stderr": 311605.46093383565, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "hendrycksTest-astronomy": { + "acc": 0.16447368421052633, + "acc_stderr": 0.030167533468632723, + "acc_norm": 0.2236842105263158, + "acc_norm_stderr": 0.033911609343436046 + }, + "winogrande": { + "acc": 0.4940805051302289, + "acc_stderr": 0.01405150083848581 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.18627450980392157, + "acc_stderr": 0.027325470966716323, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.030964517926923393 + }, + "crows_pairs_french_age": { + "likelihood_difference": 5.902777777777778, + "likelihood_difference_stderr": 0.745349522367746, + "pct_stereotype": 0.6777777777777778, + "pct_stereotype_stderr": 0.049536623805744535 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.36, + "acc_norm_stderr": 0.04824181513244218 + }, + "hendrycksTest-global_facts": { + "acc": 0.33, + "acc_stderr": 0.047258156262526045, + "acc_norm": 0.32, + "acc_norm_stderr": 0.046882617226215034 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.2052980132450331, + "acc_stderr": 0.03297986648473836, + "acc_norm": 0.2980132450331126, + "acc_norm_stderr": 0.03734535676787198 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 12.683394451530612, + "likelihood_difference_stderr": 0.8321591288729919, + "pct_stereotype": 0.45918367346938777, + "pct_stereotype_stderr": 0.03568624151230552 + }, + "hendrycksTest-international_law": { + "acc": 0.09917355371900827, + "acc_stderr": 0.027285246312758957, + "acc_norm": 0.2396694214876033, + "acc_norm_stderr": 0.03896878985070417 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.26, + "acc_stderr": 0.0440844002276808, + "acc_norm": 0.2, + "acc_norm_stderr": 0.04020151261036845 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.17177914110429449, + "acc_stderr": 0.029634717272371013, + "acc_norm": 0.25766871165644173, + "acc_norm_stderr": 0.03436150827846917 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.22254335260115607, + "acc_stderr": 0.02239421566194282, + "acc_norm": 0.21965317919075145, + "acc_norm_stderr": 0.022289638852617893 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 7.655769230769231, + "likelihood_difference_stderr": 1.2456701776455885, + "pct_stereotype": 0.6307692307692307, + "pct_stereotype_stderr": 0.060324565928300454 + }, + "hendrycksTest-prehistory": { + "acc": 0.25925925925925924, + "acc_stderr": 0.02438366553103545, + "acc_norm": 0.24382716049382716, + "acc_norm_stderr": 0.023891879541959603 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.14, + "acc_stderr": 0.0348735088019777, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036623 + }, + "crows_pairs_french": { + "likelihood_difference": 10.100835755813954, + "likelihood_difference_stderr": 0.23128974328889199, + "pct_stereotype": 0.5819916517590936, + "pct_stereotype_stderr": 0.012047969184920519 + }, + "wsc": { + "acc": 0.6346153846153846, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135303, + "acc_norm": 0.20689655172413793, + "acc_norm_stderr": 0.03375672449560554 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 6.219184027777778, + "likelihood_difference_stderr": 0.8156476562247187, + "pct_stereotype": 0.5138888888888888, + "pct_stereotype_stderr": 0.05931618532716555 + }, + "hendrycksTest-management": { + "acc": 0.1941747572815534, + "acc_stderr": 0.03916667762822582, + "acc_norm": 0.23300970873786409, + "acc_norm_stderr": 0.04185832598928315 + }, + "hendrycksTest-machine_learning": { + "acc": 0.25, + "acc_stderr": 0.04109974682633932, + "acc_norm": 0.3125, + "acc_norm_stderr": 0.043994650575715215 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 5.5294045275590555, + "likelihood_difference_stderr": 0.34271615785671483, + "pct_stereotype": 0.36811023622047245, + "pct_stereotype_stderr": 0.021419317453594672 + }, + "hendrycksTest-marketing": { + "acc": 0.2222222222222222, + "acc_stderr": 0.027236013946196666, + "acc_norm": 0.23931623931623933, + "acc_norm_stderr": 0.02795182680892433 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.20689655172413793, + "acc_stderr": 0.028501378167893946, + "acc_norm": 0.22167487684729065, + "acc_norm_stderr": 0.029225575892489617 + }, + "hendrycksTest-econometrics": { + "acc": 0.24561403508771928, + "acc_stderr": 0.0404933929774814, + "acc_norm": 0.2807017543859649, + "acc_norm_stderr": 0.04227054451232199 + }, + "hendrycksTest-virology": { + "acc": 0.14457831325301204, + "acc_stderr": 0.027377874786362316, + "acc_norm": 0.18674698795180722, + "acc_norm_stderr": 0.030338749144500615 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.22752293577981653, + "acc_stderr": 0.017974463578776502, + "acc_norm": 0.24954128440366974, + "acc_norm_stderr": 0.01855389762950162 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.25252525252525254, + "acc_stderr": 0.030954055470365897, + "acc_norm": 0.2474747474747475, + "acc_norm_stderr": 0.03074630074212451 + }, + "sciq": { + "acc": 0.223, + "acc_stderr": 0.013169830843425661, + "acc_norm": 0.21, + "acc_norm_stderr": 0.012886662332274547 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 12.11983695652174, + "likelihood_difference_stderr": 0.9761138647537818, + "pct_stereotype": 0.6608695652173913, + "pct_stereotype_stderr": 0.04433930011819816 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 4.749609375, + "likelihood_difference_stderr": 0.4877724715110692, + "pct_stereotype": 0.48125, + "pct_stereotype_stderr": 0.027974934901776306 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.26595744680851063, + "acc_stderr": 0.026358065698880582, + "acc_norm": 0.25886524822695034, + "acc_norm_stderr": 0.026129572527180848 + }, + "logiqa": { + "acc": 0.2196620583717358, + "acc_stderr": 0.01623910941493393, + "acc_norm": 0.23809523809523808, + "acc_norm_stderr": 0.016705867034419633 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.22794117647058823, + "acc_stderr": 0.025483081468029804, + "acc_norm": 0.2867647058823529, + "acc_norm_stderr": 0.027472274473233818 + }, + "hendrycksTest-world_religions": { + "acc": 0.1695906432748538, + "acc_stderr": 0.028782108105401712, + "acc_norm": 0.22807017543859648, + "acc_norm_stderr": 0.03218093795602357 + }, + "hendrycksTest-sociology": { + "acc": 0.2835820895522388, + "acc_stderr": 0.03187187537919796, + "acc_norm": 0.2935323383084577, + "acc_norm_stderr": 0.032200241045342054 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.22058823529411764, + "acc_stderr": 0.01677467236546854, + "acc_norm": 0.24019607843137256, + "acc_norm_stderr": 0.017282760695167435 + }, + "hendrycksTest-computer_security": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "acc_norm": 0.28, + "acc_norm_stderr": 0.045126085985421276 + }, + "hendrycksTest-philosophy": { + "acc": 0.2379421221864952, + "acc_stderr": 0.024185150647818707, + "acc_norm": 0.2861736334405145, + "acc_norm_stderr": 0.025670259242188943 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 9.869972826086956, + "likelihood_difference_stderr": 0.3709338879215957, + "pct_stereotype": 0.7130434782608696, + "pct_stereotype_stderr": 0.021113474740601688 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.18490566037735848, + "acc_stderr": 0.023893351834464324, + "acc_norm": 0.28679245283018867, + "acc_norm_stderr": 0.027834912527544067 + }, + "crows_pairs_english": { + "likelihood_difference": 5.480079755515802, + "likelihood_difference_stderr": 0.19151850776212573, + "pct_stereotype": 0.45855694692904, + "pct_stereotype_stderr": 0.012171273580365826 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 9.49802371541502, + "likelihood_difference_stderr": 0.5281355544781192, + "pct_stereotype": 0.4980237154150198, + "pct_stereotype_stderr": 0.031496793380453074 + }, + "hendrycksTest-nutrition": { + "acc": 0.20915032679738563, + "acc_stderr": 0.023287685312334803, + "acc_norm": 0.24836601307189543, + "acc_norm_stderr": 0.02473998135511359 + }, + "hendrycksTest-college_medicine": { + "acc": 0.19653179190751446, + "acc_stderr": 0.030299574664788147, + "acc_norm": 0.24855491329479767, + "acc_norm_stderr": 0.03295304696818318 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 5.714967105263158, + "likelihood_difference_stderr": 0.5307740830599903, + "pct_stereotype": 0.5684210526315789, + "pct_stereotype_stderr": 0.03602751443822843 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.2414772727272725, + "likelihood_difference_stderr": 2.881736459713796, + "pct_stereotype": 0.7272727272727273, + "pct_stereotype_stderr": 0.14083575804390605 + }, + "hendrycksTest-anatomy": { + "acc": 0.2074074074074074, + "acc_stderr": 0.03502553170678318, + "acc_norm": 0.28888888888888886, + "acc_norm_stderr": 0.0391545063041425 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.2037037037037037, + "acc_stderr": 0.02074274056012268, + "acc_norm": 0.21957671957671956, + "acc_norm_stderr": 0.021320018599770375 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.2, + "acc_stderr": 0.022755204959542936, + "acc_norm": 0.22580645161290322, + "acc_norm_stderr": 0.02378557788418101 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.359206989247312, + "likelihood_difference_stderr": 0.7683231947337748, + "pct_stereotype": 0.6021505376344086, + "pct_stereotype_stderr": 0.0510291122856655 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.23148148148148148, + "acc_stderr": 0.02876511171804696, + "acc_norm": 0.2962962962962963, + "acc_norm_stderr": 0.031141447823536037 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 10.003472222222221, + "likelihood_difference_stderr": 1.3633059287800664, + "pct_stereotype": 0.4861111111111111, + "pct_stereotype_stderr": 0.059316185327165566 + }, + "hendrycksTest-formal_logic": { + "acc": 0.2857142857142857, + "acc_stderr": 0.04040610178208841, + "acc_norm": 0.25396825396825395, + "acc_norm_stderr": 0.038932596106046706 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.25190839694656486, + "acc_stderr": 0.038073871163060866, + "acc_norm": 0.25190839694656486, + "acc_norm_stderr": 0.038073871163060866 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.23798882681564246, + "acc_norm_stderr": 0.014242630070574915 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "acc_norm": 0.17, + "acc_norm_stderr": 0.03775251680686371 + }, + "arc_easy": { + "acc": 0.2668350168350168, + "acc_stderr": 0.00907591585926725, + "acc_norm": 0.2638888888888889, + "acc_norm_stderr": 0.009043789220055139 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.21, + "acc_stderr": 0.04093601807403325, + "acc_norm": 0.26, + "acc_norm_stderr": 0.0440844002276808 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.18067226890756302, + "acc_stderr": 0.02499196496660074, + "acc_norm": 0.2773109243697479, + "acc_norm_stderr": 0.029079374539480007 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.18518518518518517, + "acc_stderr": 0.03755265865037181, + "acc_norm": 0.24074074074074073, + "acc_norm_stderr": 0.041331194402438376 + }, + "hendrycksTest-college_physics": { + "acc": 0.13725490196078433, + "acc_stderr": 0.03424084669891521, + "acc_norm": 0.20588235294117646, + "acc_norm_stderr": 0.04023382273617747 + }, + "hendrycksTest-public_relations": { + "acc": 0.2636363636363636, + "acc_stderr": 0.04220224692971987, + "acc_norm": 0.20909090909090908, + "acc_norm_stderr": 0.03895091015724138 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 5.728322072072072, + "likelihood_difference_stderr": 0.6965067589462834, + "pct_stereotype": 0.45045045045045046, + "pct_stereotype_stderr": 0.04743846177747609 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.2230769230769231, + "acc_stderr": 0.02110773012724399, + "acc_norm": 0.25384615384615383, + "acc_norm_stderr": 0.022066054378726257 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 15.282967032967033, + "likelihood_difference_stderr": 1.0847203102990313, + "pct_stereotype": 0.8021978021978022, + "pct_stereotype_stderr": 0.04198895203196222 + }, + "hendrycksTest-professional_law": { + "acc": 0.23533246414602346, + "acc_stderr": 0.010834432543912219, + "acc_norm": 0.25684485006518903, + "acc_norm_stderr": 0.011158455853098851 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.189873417721519, + "acc_stderr": 0.025530100460233494, + "acc_norm": 0.22362869198312235, + "acc_norm_stderr": 0.02712329820522997 + }, + "hendrycksTest-business_ethics": { + "acc": 0.26, + "acc_stderr": 0.044084400227680794, + "acc_norm": 0.29, + "acc_norm_stderr": 0.04560480215720684 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 7.216346153846154, + "likelihood_difference_stderr": 1.9704931663267538, + "pct_stereotype": 0.46153846153846156, + "pct_stereotype_stderr": 0.14390989949130545 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2723404255319149, + "acc_stderr": 0.029101290698386708, + "acc_norm": 0.25957446808510637, + "acc_norm_stderr": 0.02865917937429232 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.418269230769231, + "likelihood_difference_stderr": 0.6082631522720632, + "pct_stereotype": 0.5274725274725275, + "pct_stereotype_stderr": 0.05262501097748859 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "arc_challenge": { + "acc": 0.20477815699658702, + "acc_stderr": 0.01179254433851342, + "acc_norm": 0.2440273037542662, + "acc_norm_stderr": 0.01255144762785626 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.17575757575757575, + "acc_stderr": 0.02972094300622445, + "acc_norm": 0.22424242424242424, + "acc_norm_stderr": 0.03256866661681102 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2503192848020434, + "acc_stderr": 0.015491088951494588, + "acc_norm": 0.25287356321839083, + "acc_norm_stderr": 0.015543377313719681 + }, + "hendrycksTest-college_biology": { + "acc": 0.2569444444444444, + "acc_stderr": 0.03653946969442099, + "acc_norm": 0.25, + "acc_norm_stderr": 0.03621034121889507 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 6.14380787037037, + "likelihood_difference_stderr": 0.5217915071777064, + "pct_stereotype": 0.37037037037037035, + "pct_stereotype_stderr": 0.03293377139415191 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 7.492017133956386, + "likelihood_difference_stderr": 0.4566662635366117, + "pct_stereotype": 0.48286604361370716, + "pct_stereotype_stderr": 0.027934433698537306 + }, + "piqa": { + "acc": 0.5272034820457019, + "acc_stderr": 0.011648545262429021, + "acc_norm": 0.5261153427638737, + "acc_norm_stderr": 0.011649900854263415 + }, + "hendrycksTest-security_studies": { + "acc": 0.31020408163265306, + "acc_stderr": 0.02961345987248438, + "acc_norm": 0.19183673469387755, + "acc_norm_stderr": 0.025206963154225378 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.19170984455958548, + "acc_stderr": 0.02840895362624527, + "acc_norm": 0.24870466321243523, + "acc_norm_stderr": 0.03119584087770028 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 14.775568181818182, + "likelihood_difference_stderr": 1.4715579883772572, + "pct_stereotype": 0.3939393939393939, + "pct_stereotype_stderr": 0.06060606060606062 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.17, + "acc_norm_stderr": 0.03775251680686371 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.2074074074074074, + "acc_stderr": 0.02472071319395215, + "acc_norm": 0.26666666666666666, + "acc_norm_stderr": 0.026962424325073824 + }, + "hendrycksTest-human_aging": { + "acc": 0.273542600896861, + "acc_stderr": 0.029918586707798824, + "acc_norm": 0.27802690582959644, + "acc_norm_stderr": 0.030069584874494033 + } + }, + "versions": { + "lambada_openai": 0, + "hendrycksTest-astronomy": 0, + "winogrande": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-high_school_physics": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-moral_disputes": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-college_mathematics": 0, + "crows_pairs_french": 0, + "wsc": 0, + "hendrycksTest-electrical_engineering": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-management": 0, + "hendrycksTest-machine_learning": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-high_school_geography": 0, + "sciq": 0, + "crows_pairs_french_religion": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-professional_accounting": 0, + "logiqa": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-philosophy": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_english": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_english_socioeconomic": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-high_school_biology": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-high_school_statistics": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-abstract_algebra": 0, + "arc_easy": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-public_relations": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-business_ethics": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-conceptual_physics": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-us_foreign_policy": 0, + "arc_challenge": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-college_biology": 0, + "crows_pairs_english_nationality": 0, + "crows_pairs_french_gender": 0, + "piqa": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-human_aging": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step32", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step33000.json b/data/checkpoint_eval/eleutherai_evals/step33000.json new file mode 100644 index 0000000000000000000000000000000000000000..c6c92237f3afb4162ff21de28289de468add7f25 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step33000.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-machine_learning": { + "acc": 0.25, + "acc_stderr": 0.04109974682633932, + "acc_norm": 0.21428571428571427, + "acc_norm_stderr": 0.03894641120044793 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 5.236141304347826, + "likelihood_difference_stderr": 0.5106076448625602, + "pct_stereotype": 0.5391304347826087, + "pct_stereotype_stderr": 0.04668566114758416 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.30514705882352944, + "acc_stderr": 0.027971541370170595, + "acc_norm": 0.27205882352941174, + "acc_norm_stderr": 0.027033041151681456 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 5.860233516483516, + "likelihood_difference_stderr": 0.5089789548023154, + "pct_stereotype": 0.8131868131868132, + "pct_stereotype_stderr": 0.04108446855035881 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.24692737430167597, + "acc_stderr": 0.014422292204808835, + "acc_norm": 0.24692737430167597, + "acc_norm_stderr": 0.014422292204808835 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 6.355113636363637, + "likelihood_difference_stderr": 1.7489509473745437, + "pct_stereotype": 0.5454545454545454, + "pct_stereotype_stderr": 0.1574591643244434 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.771291208791209, + "likelihood_difference_stderr": 0.26169461121705356, + "pct_stereotype": 0.5164835164835165, + "pct_stereotype_stderr": 0.05267597952306975 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.3435114503816794, + "acc_stderr": 0.041649760719448786, + "acc_norm": 0.2900763358778626, + "acc_norm_stderr": 0.03980066246467766 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.02265625, + "likelihood_difference_stderr": 0.2641863477227852, + "pct_stereotype": 0.540625, + "pct_stereotype_stderr": 0.027902068404300068 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.1814814814814815, + "acc_stderr": 0.02349926466940731, + "acc_norm": 0.23703703703703705, + "acc_norm_stderr": 0.025928876132766104 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.28, + "acc_stderr": 0.04512608598542127, + "acc_norm": 0.27, + "acc_norm_stderr": 0.04461960433384741 + }, + "hendrycksTest-formal_logic": { + "acc": 0.2777777777777778, + "acc_stderr": 0.04006168083848876, + "acc_norm": 0.23809523809523808, + "acc_norm_stderr": 0.038095238095238126 + }, + "hendrycksTest-computer_security": { + "acc": 0.22, + "acc_stderr": 0.041633319989322716, + "acc_norm": 0.28, + "acc_norm_stderr": 0.045126085985421255 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2896551724137931, + "acc_stderr": 0.03780019230438014, + "acc_norm": 0.31724137931034485, + "acc_norm_stderr": 0.03878352372138622 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.25660377358490566, + "acc_stderr": 0.026880647889051982, + "acc_norm": 0.3320754716981132, + "acc_norm_stderr": 0.028985455652334388 + }, + "hendrycksTest-human_aging": { + "acc": 0.23766816143497757, + "acc_stderr": 0.028568079464714263, + "acc_norm": 0.21973094170403587, + "acc_norm_stderr": 0.02779017706438361 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.497311827956989, + "likelihood_difference_stderr": 0.547355686843944, + "pct_stereotype": 0.8172043010752689, + "pct_stereotype_stderr": 0.040295300106155174 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.21243523316062177, + "acc_stderr": 0.02951928261681725, + "acc_norm": 0.2694300518134715, + "acc_norm_stderr": 0.03201867122877794 + }, + "hendrycksTest-professional_law": { + "acc": 0.24185136897001303, + "acc_stderr": 0.010936550813827065, + "acc_norm": 0.288135593220339, + "acc_norm_stderr": 0.011567140661324568 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.24019607843137256, + "acc_stderr": 0.01728276069516741, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.01784808957491322 + }, + "hendrycksTest-marketing": { + "acc": 0.24786324786324787, + "acc_stderr": 0.028286324075564404, + "acc_norm": 0.2777777777777778, + "acc_norm_stderr": 0.02934311479809447 + }, + "hendrycksTest-management": { + "acc": 0.1941747572815534, + "acc_stderr": 0.03916667762822583, + "acc_norm": 0.2815533980582524, + "acc_norm_stderr": 0.04453254836326469 + }, + "hendrycksTest-public_relations": { + "acc": 0.3, + "acc_stderr": 0.04389311454644286, + "acc_norm": 0.22727272727272727, + "acc_norm_stderr": 0.040139645540727735 + }, + "hendrycksTest-nutrition": { + "acc": 0.2875816993464052, + "acc_stderr": 0.02591780611714716, + "acc_norm": 0.35947712418300654, + "acc_norm_stderr": 0.027475969910660952 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.818142361111111, + "likelihood_difference_stderr": 0.641566712133372, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.05933908290969268 + }, + "hendrycksTest-college_medicine": { + "acc": 0.21965317919075145, + "acc_stderr": 0.031568093627031744, + "acc_norm": 0.32947976878612717, + "acc_norm_stderr": 0.03583901754736412 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.48705093503937, + "likelihood_difference_stderr": 0.17608529748474508, + "pct_stereotype": 0.4625984251968504, + "pct_stereotype_stderr": 0.02214356608896984 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.20245398773006135, + "acc_stderr": 0.03157065078911902, + "acc_norm": 0.3128834355828221, + "acc_norm_stderr": 0.03642914578292404 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.23544973544973544, + "acc_stderr": 0.02185150982203172, + "acc_norm": 0.24603174603174602, + "acc_norm_stderr": 0.022182037202948365 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.24113475177304963, + "acc_stderr": 0.02551873104953776, + "acc_norm": 0.2553191489361702, + "acc_norm_stderr": 0.026011992930902013 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.22424242424242424, + "acc_stderr": 0.03256866661681102, + "acc_norm": 0.2909090909090909, + "acc_norm_stderr": 0.03546563019624337 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.17, + "acc_stderr": 0.03775251680686371, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816505 + }, + "hendrycksTest-prehistory": { + "acc": 0.2716049382716049, + "acc_stderr": 0.024748624490537382, + "acc_norm": 0.21604938271604937, + "acc_norm_stderr": 0.022899162918445785 + }, + "hendrycksTest-global_facts": { + "acc": 0.23, + "acc_stderr": 0.042295258468165065, + "acc_norm": 0.23, + "acc_norm_stderr": 0.042295258468165065 + }, + "hendrycksTest-anatomy": { + "acc": 0.16296296296296298, + "acc_stderr": 0.0319054147448284, + "acc_norm": 0.17037037037037037, + "acc_norm_stderr": 0.03247781185995593 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.807146990740741, + "likelihood_difference_stderr": 0.2792489767677307, + "pct_stereotype": 0.4351851851851852, + "pct_stereotype_stderr": 0.03381200005643525 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.25806451612903225, + "acc_stderr": 0.024892469172462826, + "acc_norm": 0.29354838709677417, + "acc_norm_stderr": 0.025906087021319295 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.8326822916666665, + "likelihood_difference_stderr": 0.42534095862131277, + "pct_stereotype": 0.5555555555555556, + "pct_stereotype_stderr": 0.05897165471491952 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.24, + "acc_stderr": 0.04292346959909284, + "acc_norm": 0.32, + "acc_norm_stderr": 0.04688261722621503 + }, + "hendrycksTest-college_physics": { + "acc": 0.21568627450980393, + "acc_stderr": 0.04092563958237655, + "acc_norm": 0.28431372549019607, + "acc_norm_stderr": 0.04488482852329017 + }, + "winogrande": { + "acc": 0.4972375690607735, + "acc_stderr": 0.014052271211616441 + }, + "logiqa": { + "acc": 0.20890937019969277, + "acc_stderr": 0.01594539939642392, + "acc_norm": 0.28417818740399386, + "acc_norm_stderr": 0.01769054268019078 + }, + "lambada_openai": { + "ppl": 118.09596009074914, + "ppl_stderr": 4.94543500156858, + "acc": 0.2233650300795653, + "acc_stderr": 0.005802673494605816 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.23232323232323232, + "acc_stderr": 0.030088629490217487, + "acc_norm": 0.2828282828282828, + "acc_norm_stderr": 0.032087795587867514 + }, + "hendrycksTest-econometrics": { + "acc": 0.23684210526315788, + "acc_stderr": 0.039994238792813365, + "acc_norm": 0.22807017543859648, + "acc_norm_stderr": 0.03947152782669415 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.586209239130435, + "likelihood_difference_stderr": 0.22439998730100816, + "pct_stereotype": 0.2847826086956522, + "pct_stereotype_stderr": 0.02106538604116979 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.9653782894736844, + "likelihood_difference_stderr": 0.28742290506987894, + "pct_stereotype": 0.6263157894736842, + "pct_stereotype_stderr": 0.035189909668609055 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.6789772727272725, + "likelihood_difference_stderr": 0.41601257841823347, + "pct_stereotype": 0.2648221343873518, + "pct_stereotype_stderr": 0.02779540983044468 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.26788990825688075, + "acc_stderr": 0.018987462257978652, + "acc_norm": 0.25871559633027524, + "acc_norm_stderr": 0.018776052319619617 + }, + "crows_pairs_french": { + "likelihood_difference": 5.2784641472868215, + "likelihood_difference_stderr": 0.12512951793875754, + "pct_stereotype": 0.407871198568873, + "pct_stereotype_stderr": 0.012004182941077525 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.4512310606060606, + "likelihood_difference_stderr": 0.6395317220387889, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.06176056549879611 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.23178807947019867, + "acc_stderr": 0.03445406271987053, + "acc_norm": 0.2119205298013245, + "acc_norm_stderr": 0.03336767086567978 + }, + "arc_easy": { + "acc": 0.4010942760942761, + "acc_stderr": 0.010057051106534374, + "acc_norm": 0.36447811447811446, + "acc_norm_stderr": 0.009875729282482438 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.25957446808510637, + "acc_stderr": 0.02865917937429232, + "acc_norm": 0.19574468085106383, + "acc_norm_stderr": 0.025937853139977145 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.49375, + "likelihood_difference_stderr": 0.48104738994215757, + "pct_stereotype": 0.4, + "pct_stereotype_stderr": 0.05192907868894985 + }, + "crows_pairs_english": { + "likelihood_difference": 3.625680158020274, + "likelihood_difference_stderr": 0.10323728907768165, + "pct_stereotype": 0.5372689326177699, + "pct_stereotype_stderr": 0.012179324068364769 + }, + "hendrycksTest-sociology": { + "acc": 0.25870646766169153, + "acc_stderr": 0.03096590312357304, + "acc_norm": 0.26865671641791045, + "acc_norm_stderr": 0.03134328358208954 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.27330779054916987, + "acc_stderr": 0.01593668106262856, + "acc_norm": 0.2503192848020434, + "acc_norm_stderr": 0.0154910889514946 + }, + "arc_challenge": { + "acc": 0.181740614334471, + "acc_stderr": 0.011269198948880236, + "acc_norm": 0.21416382252559726, + "acc_norm_stderr": 0.011988383205966497 + }, + "hendrycksTest-world_religions": { + "acc": 0.29239766081871343, + "acc_stderr": 0.03488647713457922, + "acc_norm": 0.3333333333333333, + "acc_norm_stderr": 0.03615507630310935 + }, + "hendrycksTest-astronomy": { + "acc": 0.21052631578947367, + "acc_stderr": 0.033176727875331574, + "acc_norm": 0.3618421052631579, + "acc_norm_stderr": 0.03910525752849724 + }, + "hendrycksTest-business_ethics": { + "acc": 0.36, + "acc_stderr": 0.048241815132442176, + "acc_norm": 0.33, + "acc_norm_stderr": 0.04725815626252604 + }, + "hendrycksTest-college_biology": { + "acc": 0.2847222222222222, + "acc_stderr": 0.037738099906869355, + "acc_norm": 0.2847222222222222, + "acc_norm_stderr": 0.03773809990686934 + }, + "piqa": { + "acc": 0.5984766050054406, + "acc_stderr": 0.011437324373397848, + "acc_norm": 0.5930359085963003, + "acc_norm_stderr": 0.011462093919190166 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.313473520249222, + "likelihood_difference_stderr": 0.19628650459456284, + "pct_stereotype": 0.5202492211838006, + "pct_stereotype_stderr": 0.027927918885132307 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.23109243697478993, + "acc_stderr": 0.02738140692786898, + "acc_norm": 0.29831932773109243, + "acc_norm_stderr": 0.02971914287634287 + }, + "hendrycksTest-virology": { + "acc": 0.21686746987951808, + "acc_stderr": 0.03208284450356365, + "acc_norm": 0.2469879518072289, + "acc_norm_stderr": 0.03357351982064536 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.26851851851851855, + "acc_stderr": 0.04284467968052191, + "acc_norm": 0.4074074074074074, + "acc_norm_stderr": 0.04750077341199985 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.24, + "acc_stderr": 0.04292346959909284, + "acc_norm": 0.26, + "acc_norm_stderr": 0.04408440022768078 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.187980769230769, + "likelihood_difference_stderr": 0.5880197346199485, + "pct_stereotype": 0.6153846153846154, + "pct_stereotype_stderr": 0.06081303192631497 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.24019607843137256, + "acc_stderr": 0.02998373305591361, + "acc_norm": 0.2696078431372549, + "acc_norm_stderr": 0.031145570659486782 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.21674876847290642, + "acc_stderr": 0.02899033125251624, + "acc_norm": 0.26108374384236455, + "acc_norm_stderr": 0.030903796952114468 + }, + "sciq": { + "acc": 0.664, + "acc_stderr": 0.014944140233795027, + "acc_norm": 0.576, + "acc_norm_stderr": 0.01563548747140519 + }, + "hendrycksTest-philosophy": { + "acc": 0.2057877813504823, + "acc_stderr": 0.022961339906764244, + "acc_norm": 0.28938906752411575, + "acc_norm_stderr": 0.025755865922632945 + }, + "hendrycksTest-security_studies": { + "acc": 0.30612244897959184, + "acc_stderr": 0.02950489645459597, + "acc_norm": 0.2530612244897959, + "acc_norm_stderr": 0.027833023871399683 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.258974358974359, + "acc_stderr": 0.022211106810061675, + "acc_norm": 0.28205128205128205, + "acc_norm_stderr": 0.022815813098896597 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.27167630057803466, + "acc_stderr": 0.023948512905468365, + "acc_norm": 0.32947976878612717, + "acc_norm_stderr": 0.025305258131879716 + }, + "hendrycksTest-international_law": { + "acc": 0.18181818181818182, + "acc_stderr": 0.03520893951097652, + "acc_norm": 0.4214876033057851, + "acc_norm_stderr": 0.04507732278775094 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.32, + "acc_norm_stderr": 0.046882617226215034 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.714527027027027, + "likelihood_difference_stderr": 0.39412229193840076, + "pct_stereotype": 0.6396396396396397, + "pct_stereotype_stderr": 0.04577621167070314 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 2.5288461538461537, + "likelihood_difference_stderr": 0.9157702142826863, + "pct_stereotype": 0.38461538461538464, + "pct_stereotype_stderr": 0.1404416814115811 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.270042194092827, + "acc_stderr": 0.028900721906293426, + "acc_norm": 0.270042194092827, + "acc_norm_stderr": 0.028900721906293426 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.18518518518518517, + "acc_stderr": 0.02649191472735516, + "acc_norm": 0.24537037037037038, + "acc_norm_stderr": 0.029346665094372924 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.29, + "acc_stderr": 0.04560480215720684, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 5.089205994897959, + "likelihood_difference_stderr": 0.3514259595841283, + "pct_stereotype": 0.3877551020408163, + "pct_stereotype_stderr": 0.03489185364347385 + } + }, + "versions": { + "hendrycksTest-machine_learning": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-professional_medicine": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-moral_scenarios": 0, + "crows_pairs_english_autre": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-human_aging": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-management": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-nutrition": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-college_computer_science": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-high_school_biology": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-college_physics": 0, + "winogrande": 0, + "logiqa": 0, + "lambada_openai": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-econometrics": 0, + "crows_pairs_french_race_color": 0, + "crows_pairs_english_socioeconomic": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-high_school_psychology": 0, + "crows_pairs_french": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-high_school_physics": 0, + "arc_easy": 0, + "hendrycksTest-conceptual_physics": 0, + "crows_pairs_french_age": 0, + "crows_pairs_english": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-miscellaneous": 0, + "arc_challenge": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-college_biology": 0, + "piqa": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-abstract_algebra": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-high_school_chemistry": 0, + "sciq": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-international_law": 0, + "wsc": 0, + "hendrycksTest-medical_genetics": 0, + "crows_pairs_english_religion": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-college_chemistry": 0, + "crows_pairs_french_socioeconomic": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step33000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:3", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step4.json b/data/checkpoint_eval/eleutherai_evals/step4.json new file mode 100644 index 0000000000000000000000000000000000000000..d3d36e18aacfe03ce9dcdddaf0e93ca84442c950 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step4.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-logical_fallacies": { + "acc": 0.1656441717791411, + "acc_stderr": 0.029208296231259104, + "acc_norm": 0.2392638036809816, + "acc_norm_stderr": 0.033519538795212696 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.24632352941176472, + "acc_stderr": 0.02617343857052, + "acc_norm": 0.27941176470588236, + "acc_norm_stderr": 0.02725720260611495 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.21794871794871795, + "acc_stderr": 0.020932445774463196, + "acc_norm": 0.24358974358974358, + "acc_norm_stderr": 0.02176373368417392 + }, + "hendrycksTest-global_facts": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "acc_norm": 0.27, + "acc_norm_stderr": 0.044619604333847394 + }, + "hendrycksTest-astronomy": { + "acc": 0.18421052631578946, + "acc_stderr": 0.031546980450822305, + "acc_norm": 0.23684210526315788, + "acc_norm_stderr": 0.03459777606810534 + }, + "arc_challenge": { + "acc": 0.21416382252559726, + "acc_stderr": 0.011988383205966503, + "acc_norm": 0.24573378839590443, + "acc_norm_stderr": 0.012581033453730107 + }, + "hendrycksTest-public_relations": { + "acc": 0.2545454545454545, + "acc_stderr": 0.041723430387053825, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.03694284335337798 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 6.980769230769231, + "likelihood_difference_stderr": 1.9266183337380198, + "pct_stereotype": 0.38461538461538464, + "pct_stereotype_stderr": 0.1404416814115811 + }, + "sciq": { + "acc": 0.193, + "acc_stderr": 0.012486268734370101, + "acc_norm": 0.216, + "acc_norm_stderr": 0.013019735539307789 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 6.260850694444445, + "likelihood_difference_stderr": 0.8507319313798797, + "pct_stereotype": 0.5277777777777778, + "pct_stereotype_stderr": 0.05924743948371487 + }, + "hendrycksTest-philosophy": { + "acc": 0.2315112540192926, + "acc_stderr": 0.023956532766639133, + "acc_norm": 0.2958199356913183, + "acc_norm_stderr": 0.025922371788818784 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.18867924528301888, + "acc_stderr": 0.024079995130062207, + "acc_norm": 0.2792452830188679, + "acc_norm_stderr": 0.027611163402399715 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "crows_pairs_french": { + "likelihood_difference": 10.009815705128204, + "likelihood_difference_stderr": 0.23461076167668074, + "pct_stereotype": 0.5688729874776386, + "pct_stereotype_stderr": 0.012096877040229652 + }, + "hendrycksTest-machine_learning": { + "acc": 0.3125, + "acc_stderr": 0.043994650575715215, + "acc_norm": 0.30357142857142855, + "acc_norm_stderr": 0.04364226155841044 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.833791208791209, + "likelihood_difference_stderr": 0.6710082273353779, + "pct_stereotype": 0.6593406593406593, + "pct_stereotype_stderr": 0.049956709512768704 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 6.0629934210526315, + "likelihood_difference_stderr": 0.5513724456968193, + "pct_stereotype": 0.46842105263157896, + "pct_stereotype_stderr": 0.03629703808831611 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2595419847328244, + "acc_stderr": 0.03844876139785271, + "acc_norm": 0.2595419847328244, + "acc_norm_stderr": 0.03844876139785271 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2553191489361702, + "acc_stderr": 0.0285048564705142, + "acc_norm": 0.25957446808510637, + "acc_norm_stderr": 0.02865917937429232 + }, + "hendrycksTest-nutrition": { + "acc": 0.17973856209150327, + "acc_stderr": 0.021986032182064148, + "acc_norm": 0.238562091503268, + "acc_norm_stderr": 0.024404394928087877 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.17880794701986755, + "acc_stderr": 0.031287448506007225, + "acc_norm": 0.31125827814569534, + "acc_norm_stderr": 0.03780445850526733 + }, + "hendrycksTest-college_medicine": { + "acc": 0.20809248554913296, + "acc_stderr": 0.03095289021774988, + "acc_norm": 0.2658959537572254, + "acc_norm_stderr": 0.03368762932259431 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.18, + "acc_stderr": 0.03861229196653695, + "acc_norm": 0.17, + "acc_norm_stderr": 0.03775251680686371 + }, + "hendrycksTest-world_religions": { + "acc": 0.19298245614035087, + "acc_stderr": 0.03026745755489847, + "acc_norm": 0.23391812865497075, + "acc_norm_stderr": 0.03246721765117825 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 10.067440711462451, + "likelihood_difference_stderr": 0.5576408190310371, + "pct_stereotype": 0.5652173913043478, + "pct_stereotype_stderr": 0.03122795678881643 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 12.441304347826087, + "likelihood_difference_stderr": 0.9687298652457538, + "pct_stereotype": 0.6086956521739131, + "pct_stereotype_stderr": 0.045709346351117126 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.22058823529411764, + "acc_stderr": 0.01677467236546854, + "acc_norm": 0.23202614379084968, + "acc_norm_stderr": 0.017077373377856992 + }, + "logiqa": { + "acc": 0.22887864823348694, + "acc_stderr": 0.01647810727631327, + "acc_norm": 0.24731182795698925, + "acc_norm_stderr": 0.016922842446712393 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.26, + "acc_stderr": 0.04408440022768079, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036624 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 7.706386292834891, + "likelihood_difference_stderr": 0.4543245217978988, + "pct_stereotype": 0.5358255451713395, + "pct_stereotype_stderr": 0.027879009258377087 + }, + "crows_pairs_french_age": { + "likelihood_difference": 5.891666666666667, + "likelihood_difference_stderr": 0.7707794292317623, + "pct_stereotype": 0.6444444444444445, + "pct_stereotype_stderr": 0.05074011803597719 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.23737373737373738, + "acc_stderr": 0.03031371053819888, + "acc_norm": 0.2727272727272727, + "acc_norm_stderr": 0.03173071239071724 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.25, + "acc_stderr": 0.029531221160930918, + "acc_norm": 0.2916666666666667, + "acc_norm_stderr": 0.030998666304560534 + }, + "hendrycksTest-international_law": { + "acc": 0.08264462809917356, + "acc_stderr": 0.02513538235660422, + "acc_norm": 0.21487603305785125, + "acc_norm_stderr": 0.037494924487096966 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2765957446808511, + "acc_stderr": 0.026684564340460997, + "acc_norm": 0.2624113475177305, + "acc_norm_stderr": 0.026244920349843007 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 6.454716435185185, + "likelihood_difference_stderr": 0.5388501876392606, + "pct_stereotype": 0.4537037037037037, + "pct_stereotype_stderr": 0.03395322726375797 + }, + "lambada_openai": { + "ppl": 3681578.6002349453, + "ppl_stderr": 359777.03680556506, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.23015873015873015, + "acc_stderr": 0.02167921966369314, + "acc_norm": 0.23809523809523808, + "acc_norm_stderr": 0.021935878081184763 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.1574074074074074, + "acc_stderr": 0.03520703990517965, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.042365112580946315 + }, + "hendrycksTest-professional_law": { + "acc": 0.23272490221642764, + "acc_stderr": 0.010792595553888486, + "acc_norm": 0.24185136897001303, + "acc_norm_stderr": 0.01093655081382706 + }, + "wsc": { + "acc": 0.6346153846153846, + "acc_stderr": 0.0474473339327792 + }, + "arc_easy": { + "acc": 0.27230639730639733, + "acc_stderr": 0.009134218447652661, + "acc_norm": 0.2521043771043771, + "acc_norm_stderr": 0.008910024163218188 + }, + "hendrycksTest-security_studies": { + "acc": 0.3142857142857143, + "acc_stderr": 0.02971932942241746, + "acc_norm": 0.19183673469387755, + "acc_norm_stderr": 0.025206963154225378 + }, + "hendrycksTest-business_ethics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "winogrande": { + "acc": 0.49329123914759276, + "acc_stderr": 0.014051220692330346 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.842741935483871, + "likelihood_difference_stderr": 0.7856648940316118, + "pct_stereotype": 0.44086021505376344, + "pct_stereotype_stderr": 0.051762678118979284 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 14.893543956043956, + "likelihood_difference_stderr": 1.025052251613424, + "pct_stereotype": 0.7912087912087912, + "pct_stereotype_stderr": 0.04284305206509431 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.23410404624277456, + "acc_stderr": 0.022797110278071138, + "acc_norm": 0.21098265895953758, + "acc_norm_stderr": 0.021966309947043117 + }, + "piqa": { + "acc": 0.5261153427638737, + "acc_stderr": 0.011649900854263415, + "acc_norm": 0.5217627856365615, + "acc_norm_stderr": 0.011654768618560074 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.25504587155963304, + "acc_stderr": 0.018688500856535853, + "acc_norm": 0.24587155963302754, + "acc_norm_stderr": 0.018461940968708457 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 5.00556640625, + "likelihood_difference_stderr": 0.5131457384215727, + "pct_stereotype": 0.496875, + "pct_stereotype_stderr": 0.02799407877242281 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.2129032258064516, + "acc_stderr": 0.02328766512726855, + "acc_norm": 0.22258064516129034, + "acc_norm_stderr": 0.02366421667164251 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 5.941806102362205, + "likelihood_difference_stderr": 0.3648000613371286, + "pct_stereotype": 0.36220472440944884, + "pct_stereotype_stderr": 0.021345863339438092 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.17575757575757575, + "acc_stderr": 0.02972094300622445, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.030117688929503585 + }, + "hendrycksTest-college_biology": { + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "acc_norm": 0.2638888888888889, + "acc_norm_stderr": 0.03685651095897532 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 6.261363636363637, + "likelihood_difference_stderr": 3.1768630246968392, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "hendrycksTest-virology": { + "acc": 0.1927710843373494, + "acc_stderr": 0.030709824050565274, + "acc_norm": 0.22289156626506024, + "acc_norm_stderr": 0.03240004825594688 + }, + "hendrycksTest-sociology": { + "acc": 0.25870646766169153, + "acc_stderr": 0.030965903123573044, + "acc_norm": 0.2736318407960199, + "acc_norm_stderr": 0.03152439186555402 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.17, + "acc_stderr": 0.0377525168068637, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036623 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.24074074074074073, + "acc_stderr": 0.026067159222275794, + "acc_norm": 0.2814814814814815, + "acc_norm_stderr": 0.027420019350945263 + }, + "hendrycksTest-human_aging": { + "acc": 0.2825112107623318, + "acc_stderr": 0.03021683101150878, + "acc_norm": 0.273542600896861, + "acc_norm_stderr": 0.029918586707798817 + }, + "hendrycksTest-marketing": { + "acc": 0.23504273504273504, + "acc_stderr": 0.027778835904935437, + "acc_norm": 0.2564102564102564, + "acc_norm_stderr": 0.028605953702004243 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.21761658031088082, + "acc_stderr": 0.029778663037752964, + "acc_norm": 0.24352331606217617, + "acc_norm_stderr": 0.030975436386845436 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.22660098522167488, + "acc_stderr": 0.029454863835292996, + "acc_norm": 0.21674876847290642, + "acc_norm_stderr": 0.028990331252516235 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.22, + "acc_norm_stderr": 0.041633319989322695 + }, + "hendrycksTest-anatomy": { + "acc": 0.21481481481481482, + "acc_stderr": 0.03547854198560823, + "acc_norm": 0.2962962962962963, + "acc_norm_stderr": 0.03944624162501117 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.19327731092436976, + "acc_stderr": 0.02564947026588919, + "acc_norm": 0.2815126050420168, + "acc_norm_stderr": 0.029213549414372163 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2, + "acc_stderr": 0.0333333333333333, + "acc_norm": 0.19310344827586207, + "acc_norm_stderr": 0.03289445522127401 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 8.886277173913044, + "likelihood_difference_stderr": 0.37545508976604897, + "pct_stereotype": 0.6391304347826087, + "pct_stereotype_stderr": 0.022416279590272897 + }, + "hendrycksTest-econometrics": { + "acc": 0.24561403508771928, + "acc_stderr": 0.04049339297748139, + "acc_norm": 0.30701754385964913, + "acc_norm_stderr": 0.0433913832257986 + }, + "hendrycksTest-computer_security": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 10.143229166666666, + "likelihood_difference_stderr": 1.3631055624376087, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.05933908290969268 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 13.071548150510203, + "likelihood_difference_stderr": 0.8662924629179967, + "pct_stereotype": 0.413265306122449, + "pct_stereotype_stderr": 0.035262902194360866 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 5.908783783783784, + "likelihood_difference_stderr": 0.7151612654576878, + "pct_stereotype": 0.5135135135135135, + "pct_stereotype_stderr": 0.04765571461988585 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 7.940384615384615, + "likelihood_difference_stderr": 1.24666655192116, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.06231481440776789 + }, + "hendrycksTest-formal_logic": { + "acc": 0.24603174603174602, + "acc_stderr": 0.03852273364924318, + "acc_norm": 0.23809523809523808, + "acc_norm_stderr": 0.03809523809523809 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 14.826704545454545, + "likelihood_difference_stderr": 1.4750985182608785, + "pct_stereotype": 0.3484848484848485, + "pct_stereotype_stderr": 0.0591013677911929 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.18627450980392157, + "acc_stderr": 0.027325470966716326, + "acc_norm": 0.2696078431372549, + "acc_norm_stderr": 0.031145570659486782 + }, + "crows_pairs_english": { + "likelihood_difference": 5.814139833035182, + "likelihood_difference_stderr": 0.20053789227207508, + "pct_stereotype": 0.456768038163387, + "pct_stereotype_stderr": 0.012167560197793074 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "acc_norm": 0.33, + "acc_norm_stderr": 0.04725815626252606 + }, + "hendrycksTest-prehistory": { + "acc": 0.23148148148148148, + "acc_stderr": 0.02346842983245115, + "acc_norm": 0.20679012345679013, + "acc_norm_stderr": 0.022535006705942818 + }, + "hendrycksTest-college_physics": { + "acc": 0.23529411764705882, + "acc_stderr": 0.04220773659171452, + "acc_norm": 0.22549019607843138, + "acc_norm_stderr": 0.04158307533083286 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2503192848020434, + "acc_stderr": 0.01549108895149458, + "acc_norm": 0.26309067688378035, + "acc_norm_stderr": 0.015745497169049053 + }, + "hendrycksTest-management": { + "acc": 0.2524271844660194, + "acc_stderr": 0.04301250399690879, + "acc_norm": 0.2912621359223301, + "acc_norm_stderr": 0.044986763205729245 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.17721518987341772, + "acc_stderr": 0.024856364184503217, + "acc_norm": 0.21518987341772153, + "acc_norm_stderr": 0.02675082699467616 + } + }, + "versions": { + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-astronomy": 0, + "arc_challenge": 0, + "hendrycksTest-public_relations": 0, + "crows_pairs_french_autre": 0, + "sciq": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-us_foreign_policy": 0, + "crows_pairs_french": 0, + "hendrycksTest-machine_learning": 0, + "crows_pairs_english_age": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_french_nationality": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-professional_psychology": 0, + "logiqa": 0, + "hendrycksTest-college_computer_science": 0, + "crows_pairs_french_gender": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-professional_accounting": 0, + "crows_pairs_english_nationality": 0, + "lambada_openai": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-professional_law": 0, + "wsc": 0, + "arc_easy": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-business_ethics": 0, + "winogrande": 0, + "crows_pairs_english_sexual_orientation": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-moral_disputes": 0, + "piqa": 0, + "hendrycksTest-high_school_psychology": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-high_school_biology": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-college_biology": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-electrical_engineering": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-computer_security": 0, + "crows_pairs_french_physical_appearance": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-medical_genetics": 0, + "crows_pairs_english_religion": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-formal_logic": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_english": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-management": 0, + "hendrycksTest-high_school_world_history": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step4", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step43000.json b/data/checkpoint_eval/eleutherai_evals/step43000.json new file mode 100644 index 0000000000000000000000000000000000000000..60781e8694a6dd8f09d6574d052a15c8474adbf7 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step43000.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-prehistory": { + "acc": 0.30246913580246915, + "acc_stderr": 0.025557653981868038, + "acc_norm": 0.21296296296296297, + "acc_norm_stderr": 0.022779719088733396 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.26011560693641617, + "acc_stderr": 0.02361867831006937, + "acc_norm": 0.30057803468208094, + "acc_norm_stderr": 0.024685316867257792 + }, + "arc_easy": { + "acc": 0.39941077441077444, + "acc_stderr": 0.010050018228742104, + "acc_norm": 0.35984848484848486, + "acc_norm_stderr": 0.009848484848484853 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.29770992366412213, + "acc_stderr": 0.040103589424622034, + "acc_norm": 0.2595419847328244, + "acc_norm_stderr": 0.03844876139785271 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.420530913978495, + "likelihood_difference_stderr": 0.5455347845945085, + "pct_stereotype": 0.8172043010752689, + "pct_stereotype_stderr": 0.04029530010615517 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2170212765957447, + "acc_stderr": 0.02694748312149623, + "acc_norm": 0.18723404255319148, + "acc_norm_stderr": 0.025501588341883583 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.1190711462450595, + "likelihood_difference_stderr": 0.4052713659970057, + "pct_stereotype": 0.2924901185770751, + "pct_stereotype_stderr": 0.02865639690849427 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.6803385416666665, + "likelihood_difference_stderr": 0.2726355174224698, + "pct_stereotype": 0.4398148148148148, + "pct_stereotype_stderr": 0.03385177976044811 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.26595744680851063, + "acc_stderr": 0.026358065698880585, + "acc_norm": 0.2375886524822695, + "acc_norm_stderr": 0.025389512552729906 + }, + "hendrycksTest-security_studies": { + "acc": 0.3346938775510204, + "acc_stderr": 0.030209235226242307, + "acc_norm": 0.2693877551020408, + "acc_norm_stderr": 0.02840125202902294 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.18, + "acc_stderr": 0.03861229196653697, + "acc_norm": 0.27, + "acc_norm_stderr": 0.044619604333847394 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.21296296296296297, + "acc_stderr": 0.0395783547198098, + "acc_norm": 0.35185185185185186, + "acc_norm_stderr": 0.04616631111801713 + }, + "piqa": { + "acc": 0.5903155603917302, + "acc_stderr": 0.011473932007187606, + "acc_norm": 0.5892274211099021, + "acc_norm_stderr": 0.011478565556775776 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.23178807947019867, + "acc_stderr": 0.03445406271987054, + "acc_norm": 0.25165562913907286, + "acc_norm_stderr": 0.03543304234389985 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.689470720720721, + "likelihood_difference_stderr": 0.45650051870103214, + "pct_stereotype": 0.6576576576576577, + "pct_stereotype_stderr": 0.04524117824423198 + }, + "hendrycksTest-computer_security": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542127 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.23870967741935484, + "acc_stderr": 0.02425107126220884, + "acc_norm": 0.3, + "acc_norm_stderr": 0.026069362295335123 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.24102564102564103, + "acc_stderr": 0.021685546665333184, + "acc_norm": 0.28717948717948716, + "acc_norm_stderr": 0.022939925418530616 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.2727272727272727, + "acc_stderr": 0.03173071239071724, + "acc_norm": 0.3181818181818182, + "acc_norm_stderr": 0.0331847733384533 + }, + "hendrycksTest-college_medicine": { + "acc": 0.2254335260115607, + "acc_stderr": 0.03186209851641143, + "acc_norm": 0.3179190751445087, + "acc_norm_stderr": 0.0355068398916558 + }, + "crows_pairs_english": { + "likelihood_difference": 3.5890820661896243, + "likelihood_difference_stderr": 0.10115312974643073, + "pct_stereotype": 0.5491949910554562, + "pct_stereotype_stderr": 0.012154039490138224 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.323863636363637, + "likelihood_difference_stderr": 1.31611564166353, + "pct_stereotype": 0.5454545454545454, + "pct_stereotype_stderr": 0.1574591643244434 + }, + "lambada_openai": { + "ppl": 112.36318862751271, + "ppl_stderr": 4.5861160676234745, + "acc": 0.22821657287017272, + "acc_stderr": 0.005847003943226629 + }, + "hendrycksTest-world_religions": { + "acc": 0.2807017543859649, + "acc_stderr": 0.034462962170884265, + "acc_norm": 0.3391812865497076, + "acc_norm_stderr": 0.03631053496488905 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.20725388601036268, + "acc_stderr": 0.029252823291803648, + "acc_norm": 0.2694300518134715, + "acc_norm_stderr": 0.03201867122877793 + }, + "hendrycksTest-college_biology": { + "acc": 0.2847222222222222, + "acc_stderr": 0.03773809990686936, + "acc_norm": 0.2916666666666667, + "acc_norm_stderr": 0.038009680605548574 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.679470486111111, + "likelihood_difference_stderr": 0.4176954857537189, + "pct_stereotype": 0.5972222222222222, + "pct_stereotype_stderr": 0.05820650942569533 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2822477650063857, + "acc_stderr": 0.016095302969878555, + "acc_norm": 0.2771392081736909, + "acc_norm_stderr": 0.01600563629412242 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.8324175824175826, + "likelihood_difference_stderr": 0.2900137402487036, + "pct_stereotype": 0.45054945054945056, + "pct_stereotype_stderr": 0.052446231001012276 + }, + "hendrycksTest-professional_law": { + "acc": 0.24967405475880053, + "acc_stderr": 0.011054538377832322, + "acc_norm": 0.28096479791395046, + "acc_norm_stderr": 0.011479684550077689 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.460503472222222, + "likelihood_difference_stderr": 0.5691473963017525, + "pct_stereotype": 0.4722222222222222, + "pct_stereotype_stderr": 0.05924743948371486 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.27, + "acc_stderr": 0.044619604333847394, + "acc_norm": 0.36, + "acc_norm_stderr": 0.048241815132442176 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542127 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.002840909090909, + "likelihood_difference_stderr": 0.6010893644517457, + "pct_stereotype": 0.3787878787878788, + "pct_stereotype_stderr": 0.0601674102524024 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.24528301886792453, + "acc_stderr": 0.02648035717989568, + "acc_norm": 0.33962264150943394, + "acc_norm_stderr": 0.029146904747798335 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 4.15016447368421, + "likelihood_difference_stderr": 0.27969083115380783, + "pct_stereotype": 0.6789473684210526, + "pct_stereotype_stderr": 0.03396059335824887 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.15925925925925927, + "acc_stderr": 0.02231039463004062, + "acc_norm": 0.23333333333333334, + "acc_norm_stderr": 0.025787874220959316 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 4.891063456632653, + "likelihood_difference_stderr": 0.3589199389598812, + "pct_stereotype": 0.4387755102040816, + "pct_stereotype_stderr": 0.035536298657903934 + }, + "winogrande": { + "acc": 0.5027624309392266, + "acc_stderr": 0.014052271211616438 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.24867724867724866, + "acc_stderr": 0.022261817692400168, + "acc_norm": 0.2698412698412698, + "acc_norm_stderr": 0.02286083830923207 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.27941176470588236, + "acc_stderr": 0.027257202606114944, + "acc_norm": 0.27205882352941174, + "acc_norm_stderr": 0.027033041151681456 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.22, + "acc_stderr": 0.041633319989322695, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "arc_challenge": { + "acc": 0.17235494880546076, + "acc_stderr": 0.011037113093461295, + "acc_norm": 0.20477815699658702, + "acc_norm_stderr": 0.011792544338513403 + }, + "hendrycksTest-anatomy": { + "acc": 0.2074074074074074, + "acc_stderr": 0.03502553170678318, + "acc_norm": 0.22962962962962963, + "acc_norm_stderr": 0.03633384414073462 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.23030303030303031, + "acc_stderr": 0.03287666758603489, + "acc_norm": 0.2909090909090909, + "acc_norm_stderr": 0.03546563019624336 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.747916666666667, + "likelihood_difference_stderr": 0.5033904535606084, + "pct_stereotype": 0.45555555555555555, + "pct_stereotype_stderr": 0.05279009646630345 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2620689655172414, + "acc_stderr": 0.036646663372252565, + "acc_norm": 0.2896551724137931, + "acc_norm_stderr": 0.03780019230438014 + }, + "hendrycksTest-philosophy": { + "acc": 0.2057877813504823, + "acc_stderr": 0.022961339906764237, + "acc_norm": 0.2958199356913183, + "acc_norm_stderr": 0.025922371788818784 + }, + "logiqa": { + "acc": 0.2304147465437788, + "acc_stderr": 0.016516834820590964, + "acc_norm": 0.2887864823348694, + "acc_norm_stderr": 0.017775906336539225 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206845 + }, + "hendrycksTest-virology": { + "acc": 0.18674698795180722, + "acc_stderr": 0.030338749144500583, + "acc_norm": 0.23493975903614459, + "acc_norm_stderr": 0.03300533186128922 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.27450980392156865, + "acc_stderr": 0.03132179803083291, + "acc_norm": 0.28921568627450983, + "acc_norm_stderr": 0.031822318676475524 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 5.036141304347826, + "likelihood_difference_stderr": 0.4580008943372021, + "pct_stereotype": 0.6086956521739131, + "pct_stereotype_stderr": 0.045709346351117126 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23910614525139665, + "acc_stderr": 0.014265554192331149, + "acc_norm": 0.24692737430167597, + "acc_norm_stderr": 0.014422292204808835 + }, + "hendrycksTest-econometrics": { + "acc": 0.21929824561403508, + "acc_stderr": 0.03892431106518754, + "acc_norm": 0.24561403508771928, + "acc_norm_stderr": 0.040493392977481425 + }, + "hendrycksTest-international_law": { + "acc": 0.18181818181818182, + "acc_stderr": 0.03520893951097653, + "acc_norm": 0.39669421487603307, + "acc_norm_stderr": 0.04465869780531009 + }, + "hendrycksTest-public_relations": { + "acc": 0.3, + "acc_stderr": 0.04389311454644286, + "acc_norm": 0.2, + "acc_norm_stderr": 0.03831305140884603 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.23039215686274508, + "acc_stderr": 0.017035229258034038, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.017848089574913226 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.19704433497536947, + "acc_stderr": 0.027986724666736212, + "acc_norm": 0.2955665024630542, + "acc_norm_stderr": 0.032104944337514575 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 5.629464285714286, + "likelihood_difference_stderr": 0.45577446371656316, + "pct_stereotype": 0.7692307692307693, + "pct_stereotype_stderr": 0.04441155916843277 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.20833333333333334, + "acc_stderr": 0.027696910713093936, + "acc_norm": 0.23148148148148148, + "acc_norm_stderr": 0.028765111718046955 + }, + "hendrycksTest-global_facts": { + "acc": 0.2, + "acc_stderr": 0.04020151261036847, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816507 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 3.5528846153846154, + "likelihood_difference_stderr": 1.1518474776945522, + "pct_stereotype": 0.3076923076923077, + "pct_stereotype_stderr": 0.13323467750529824 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 4.724038461538462, + "likelihood_difference_stderr": 0.5305206431151923, + "pct_stereotype": 0.6461538461538462, + "pct_stereotype_stderr": 0.05977027026123099 + }, + "sciq": { + "acc": 0.663, + "acc_stderr": 0.014955087918653605, + "acc_norm": 0.586, + "acc_norm_stderr": 0.015583544104177506 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.2, + "acc_stderr": 0.040201512610368445, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036623 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.19747899159663865, + "acc_stderr": 0.02585916412205145, + "acc_norm": 0.29411764705882354, + "acc_norm_stderr": 0.029597329730978103 + }, + "hendrycksTest-marketing": { + "acc": 0.2777777777777778, + "acc_stderr": 0.02934311479809447, + "acc_norm": 0.2905982905982906, + "acc_norm_stderr": 0.029745048572674054 + }, + "hendrycksTest-formal_logic": { + "acc": 0.31746031746031744, + "acc_stderr": 0.04163453031302859, + "acc_norm": 0.2857142857142857, + "acc_norm_stderr": 0.0404061017820884 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "hendrycksTest-business_ethics": { + "acc": 0.34, + "acc_stderr": 0.04760952285695235, + "acc_norm": 0.29, + "acc_norm_stderr": 0.04560480215720684 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.25153374233128833, + "acc_stderr": 0.034089978868575295, + "acc_norm": 0.3312883435582822, + "acc_norm_stderr": 0.03697983910025588 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.22784810126582278, + "acc_stderr": 0.027303484599069432, + "acc_norm": 0.27848101265822783, + "acc_norm_stderr": 0.029178682304842548 + }, + "crows_pairs_french": { + "likelihood_difference": 5.100435114788312, + "likelihood_difference_stderr": 0.12283143626277805, + "pct_stereotype": 0.4263565891472868, + "pct_stereotype_stderr": 0.012080098824602488 + }, + "hendrycksTest-human_aging": { + "acc": 0.273542600896861, + "acc_stderr": 0.029918586707798834, + "acc_norm": 0.2600896860986547, + "acc_norm_stderr": 0.029442495585857473 + }, + "hendrycksTest-machine_learning": { + "acc": 0.3392857142857143, + "acc_stderr": 0.04493949068613539, + "acc_norm": 0.24107142857142858, + "acc_norm_stderr": 0.04059867246952688 + }, + "hendrycksTest-astronomy": { + "acc": 0.17105263157894737, + "acc_stderr": 0.03064360707167709, + "acc_norm": 0.32894736842105265, + "acc_norm_stderr": 0.038234289699266046 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.450602854330709, + "likelihood_difference_stderr": 0.17629031976839826, + "pct_stereotype": 0.4704724409448819, + "pct_stereotype_stderr": 0.022167024359332235 + }, + "hendrycksTest-management": { + "acc": 0.21359223300970873, + "acc_stderr": 0.04058042015646034, + "acc_norm": 0.3106796116504854, + "acc_norm_stderr": 0.04582124160161551 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.47445652173913, + "likelihood_difference_stderr": 0.22998239662821754, + "pct_stereotype": 0.2956521739130435, + "pct_stereotype_stderr": 0.021299910806810252 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.23486238532110093, + "acc_stderr": 0.018175110510343578, + "acc_norm": 0.24954128440366974, + "acc_norm_stderr": 0.018553897629501617 + }, + "hendrycksTest-college_physics": { + "acc": 0.20588235294117646, + "acc_stderr": 0.04023382273617747, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.043898699568087785 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.302570093457944, + "likelihood_difference_stderr": 0.20681513943922145, + "pct_stereotype": 0.5451713395638629, + "pct_stereotype_stderr": 0.027836551402899614 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.0423828125, + "likelihood_difference_stderr": 0.24361353048777595, + "pct_stereotype": 0.553125, + "pct_stereotype_stderr": 0.027836160509246817 + }, + "hendrycksTest-sociology": { + "acc": 0.2885572139303483, + "acc_stderr": 0.0320384104021332, + "acc_norm": 0.31840796019900497, + "acc_norm_stderr": 0.03294118479054095 + }, + "hendrycksTest-nutrition": { + "acc": 0.24836601307189543, + "acc_stderr": 0.024739981355113592, + "acc_norm": 0.34967320261437906, + "acc_norm_stderr": 0.0273053080762747 + } + }, + "versions": { + "hendrycksTest-prehistory": 0, + "hendrycksTest-moral_disputes": 0, + "arc_easy": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-conceptual_physics": 0, + "crows_pairs_french_nationality": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-jurisprudence": 0, + "piqa": 0, + "hendrycksTest-high_school_physics": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_english": 0, + "crows_pairs_english_autre": 0, + "lambada_openai": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-college_biology": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-miscellaneous": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-professional_law": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-college_chemistry": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-high_school_mathematics": 0, + "crows_pairs_french_socioeconomic": 0, + "winogrande": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-us_foreign_policy": 0, + "wsc": 0, + "arc_challenge": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-high_school_european_history": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-philosophy": 0, + "logiqa": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-high_school_chemistry": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-global_facts": 0, + "crows_pairs_french_autre": 0, + "crows_pairs_english_disability": 0, + "sciq": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-high_school_world_history": 0, + "crows_pairs_french": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-astronomy": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-management": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-college_physics": 0, + "crows_pairs_french_gender": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-nutrition": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step43000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:4", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step512.json b/data/checkpoint_eval/eleutherai_evals/step512.json new file mode 100644 index 0000000000000000000000000000000000000000..9bb5d8b330c237274269416d9b32f1dc30b5b6e9 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step512.json @@ -0,0 +1,622 @@ +{ + "results": { + "crows_pairs_french": { + "likelihood_difference": 6.341835308586762, + "likelihood_difference_stderr": 0.15048967426229426, + "pct_stereotype": 0.49314251639833034, + "pct_stereotype_stderr": 0.012212150501851282 + }, + "hendrycksTest-virology": { + "acc": 0.2289156626506024, + "acc_stderr": 0.03270745277352477, + "acc_norm": 0.24096385542168675, + "acc_norm_stderr": 0.03329394119073528 + }, + "hendrycksTest-econometrics": { + "acc": 0.2631578947368421, + "acc_stderr": 0.0414243971948936, + "acc_norm": 0.2982456140350877, + "acc_norm_stderr": 0.04303684033537315 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.23669724770642203, + "acc_stderr": 0.01822407811729908, + "acc_norm": 0.25137614678899084, + "acc_norm_stderr": 0.018599206360287415 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 4.267434210526316, + "likelihood_difference_stderr": 0.33123322148424716, + "pct_stereotype": 0.5210526315789473, + "pct_stereotype_stderr": 0.03633739504773335 + }, + "hendrycksTest-security_studies": { + "acc": 0.3224489795918367, + "acc_stderr": 0.029923100563683913, + "acc_norm": 0.2, + "acc_norm_stderr": 0.025607375986579157 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.2128205128205128, + "acc_stderr": 0.020752423722128006, + "acc_norm": 0.24615384615384617, + "acc_norm_stderr": 0.021840866990423088 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.94481981981982, + "likelihood_difference_stderr": 0.43861537266600115, + "pct_stereotype": 0.5135135135135135, + "pct_stereotype_stderr": 0.04765571461988585 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.23737373737373738, + "acc_stderr": 0.030313710538198892, + "acc_norm": 0.26262626262626265, + "acc_norm_stderr": 0.03135305009533087 + }, + "crows_pairs_english": { + "likelihood_difference": 4.10218209600477, + "likelihood_difference_stderr": 0.1236077959409224, + "pct_stereotype": 0.48598688133571855, + "pct_stereotype_stderr": 0.012208501686447064 + }, + "logiqa": { + "acc": 0.19201228878648233, + "acc_stderr": 0.01544934998590095, + "acc_norm": 0.22427035330261136, + "acc_norm_stderr": 0.016360043348265515 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.22794117647058823, + "acc_stderr": 0.025483081468029804, + "acc_norm": 0.22426470588235295, + "acc_norm_stderr": 0.025336848563332348 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.22254335260115607, + "acc_stderr": 0.02239421566194282, + "acc_norm": 0.17630057803468208, + "acc_norm_stderr": 0.02051642567249071 + }, + "crows_pairs_french_age": { + "likelihood_difference": 3.9694444444444446, + "likelihood_difference_stderr": 0.42986839067167343, + "pct_stereotype": 0.4444444444444444, + "pct_stereotype_stderr": 0.052671718126664185 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.424454828660436, + "likelihood_difference_stderr": 0.22752081870537358, + "pct_stereotype": 0.5389408099688473, + "pct_stereotype_stderr": 0.027865952192986033 + }, + "hendrycksTest-college_biology": { + "acc": 0.2222222222222222, + "acc_stderr": 0.034765901043041336, + "acc_norm": 0.24305555555555555, + "acc_norm_stderr": 0.03586879280080342 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.2185430463576159, + "acc_stderr": 0.03374235550425694, + "acc_norm": 0.25165562913907286, + "acc_norm_stderr": 0.035433042343899844 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 6.619357638888889, + "likelihood_difference_stderr": 0.8210291220143636, + "pct_stereotype": 0.5555555555555556, + "pct_stereotype_stderr": 0.05897165471491952 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 8.846590909090908, + "likelihood_difference_stderr": 0.9260560084979663, + "pct_stereotype": 0.4090909090909091, + "pct_stereotype_stderr": 0.060983672113630656 + }, + "hendrycksTest-global_facts": { + "acc": 0.26, + "acc_stderr": 0.04408440022768079, + "acc_norm": 0.27, + "acc_norm_stderr": 0.044619604333847394 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 6.335817920918367, + "likelihood_difference_stderr": 0.49605368420562174, + "pct_stereotype": 0.3877551020408163, + "pct_stereotype_stderr": 0.03489185364347385 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.18518518518518517, + "acc_stderr": 0.03755265865037181, + "acc_norm": 0.18518518518518517, + "acc_norm_stderr": 0.03755265865037182 + }, + "hendrycksTest-machine_learning": { + "acc": 0.24107142857142858, + "acc_stderr": 0.04059867246952686, + "acc_norm": 0.2767857142857143, + "acc_norm_stderr": 0.042466243366976256 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.14074074074074075, + "acc_stderr": 0.0212029303435688, + "acc_norm": 0.2, + "acc_norm_stderr": 0.024388430433987664 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2900763358778626, + "acc_stderr": 0.03980066246467765, + "acc_norm": 0.31297709923664124, + "acc_norm_stderr": 0.04066962905677698 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.23, + "acc_norm_stderr": 0.042295258468165044 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.18652849740932642, + "acc_stderr": 0.02811209121011746, + "acc_norm": 0.2538860103626943, + "acc_norm_stderr": 0.03141024780565319 + }, + "hendrycksTest-professional_law": { + "acc": 0.2379400260756193, + "acc_stderr": 0.010875700787694231, + "acc_norm": 0.26988265971316816, + "acc_norm_stderr": 0.011337381084250423 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.2, + "acc_stderr": 0.024618298195866518, + "acc_norm": 0.2943396226415094, + "acc_norm_stderr": 0.028049186315695245 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.20689655172413793, + "acc_stderr": 0.02850137816789395, + "acc_norm": 0.2512315270935961, + "acc_norm_stderr": 0.030516530732694436 + }, + "hendrycksTest-management": { + "acc": 0.22330097087378642, + "acc_stderr": 0.04123553189891431, + "acc_norm": 0.24271844660194175, + "acc_norm_stderr": 0.04245022486384495 + }, + "hendrycksTest-astronomy": { + "acc": 0.21710526315789475, + "acc_stderr": 0.03355045304882924, + "acc_norm": 0.2894736842105263, + "acc_norm_stderr": 0.03690677986137282 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.1393939393939394, + "acc_stderr": 0.0270459488258654, + "acc_norm": 0.23636363636363636, + "acc_norm_stderr": 0.03317505930009182 + }, + "hendrycksTest-college_medicine": { + "acc": 0.24277456647398843, + "acc_stderr": 0.0326926380614177, + "acc_norm": 0.2254335260115607, + "acc_norm_stderr": 0.03186209851641143 + }, + "hendrycksTest-college_physics": { + "acc": 0.16666666666666666, + "acc_stderr": 0.03708284662416542, + "acc_norm": 0.24509803921568626, + "acc_norm_stderr": 0.042801058373643966 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.21008403361344538, + "acc_stderr": 0.026461398717471874, + "acc_norm": 0.27310924369747897, + "acc_norm_stderr": 0.02894200404099817 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.7877604166666665, + "likelihood_difference_stderr": 0.4734800383795231, + "pct_stereotype": 0.5138888888888888, + "pct_stereotype_stderr": 0.05931618532716555 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.19, + "acc_stderr": 0.03942772444036625, + "acc_norm": 0.2, + "acc_norm_stderr": 0.04020151261036843 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.2275132275132275, + "acc_stderr": 0.021591269407823774, + "acc_norm": 0.2222222222222222, + "acc_norm_stderr": 0.021411684393694185 + }, + "hendrycksTest-human_aging": { + "acc": 0.3452914798206278, + "acc_stderr": 0.03191100192835794, + "acc_norm": 0.2825112107623318, + "acc_norm_stderr": 0.030216831011508773 + }, + "winogrande": { + "acc": 0.5098658247829518, + "acc_stderr": 0.014049749833367592 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 6.442788461538462, + "likelihood_difference_stderr": 0.7741982131043712, + "pct_stereotype": 0.5230769230769231, + "pct_stereotype_stderr": 0.06243339646441512 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 14.371565934065934, + "likelihood_difference_stderr": 1.0132795779676502, + "pct_stereotype": 0.8021978021978022, + "pct_stereotype_stderr": 0.04198895203196222 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.22258064516129034, + "acc_stderr": 0.023664216671642514, + "acc_norm": 0.24516129032258063, + "acc_norm_stderr": 0.024472243840895525 + }, + "arc_challenge": { + "acc": 0.18003412969283278, + "acc_stderr": 0.011227856729050028, + "acc_norm": 0.2175767918088737, + "acc_norm_stderr": 0.012057262020972499 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.24, + "acc_stderr": 0.04292346959909282, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036624 + }, + "lambada_openai": { + "ppl": 116756.33428953367, + "ppl_stderr": 6456.789280142739, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.444220430107527, + "likelihood_difference_stderr": 0.6472437756111237, + "pct_stereotype": 0.7096774193548387, + "pct_stereotype_stderr": 0.04732351421824121 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.25462962962962965, + "acc_stderr": 0.02971127586000536, + "acc_norm": 0.24074074074074073, + "acc_norm_stderr": 0.029157522184605596 + }, + "hendrycksTest-computer_security": { + "acc": 0.27, + "acc_stderr": 0.04461960433384739, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.9554347826086955, + "likelihood_difference_stderr": 0.22275405195298537, + "pct_stereotype": 0.5260869565217391, + "pct_stereotype_stderr": 0.0233062153668594 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.65625, + "likelihood_difference_stderr": 0.4765308636339587, + "pct_stereotype": 0.38461538461538464, + "pct_stereotype_stderr": 0.051282051282051246 + }, + "hendrycksTest-philosophy": { + "acc": 0.22186495176848875, + "acc_stderr": 0.02359885829286305, + "acc_norm": 0.2797427652733119, + "acc_norm_stderr": 0.0254942593506949 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2827586206896552, + "acc_stderr": 0.037528339580033376, + "acc_norm": 0.2689655172413793, + "acc_norm_stderr": 0.036951833116502325 + }, + "hendrycksTest-sociology": { + "acc": 0.21393034825870647, + "acc_stderr": 0.028996909693328927, + "acc_norm": 0.21393034825870647, + "acc_norm_stderr": 0.02899690969332891 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2695035460992908, + "acc_stderr": 0.026469036818590624, + "acc_norm": 0.2730496453900709, + "acc_norm_stderr": 0.02657786094330786 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.24393358876117496, + "acc_stderr": 0.01535721266582948, + "acc_norm": 0.25287356321839083, + "acc_norm_stderr": 0.015543377313719681 + }, + "sciq": { + "acc": 0.264, + "acc_stderr": 0.01394627184944047, + "acc_norm": 0.275, + "acc_norm_stderr": 0.014127086556490528 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 5.8173076923076925, + "likelihood_difference_stderr": 0.8524880376227814, + "pct_stereotype": 0.3076923076923077, + "pct_stereotype_stderr": 0.13323467750529824 + }, + "hendrycksTest-public_relations": { + "acc": 0.3181818181818182, + "acc_stderr": 0.044612721759105085, + "acc_norm": 0.15454545454545454, + "acc_norm_stderr": 0.03462262571262667 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 7.920380434782609, + "likelihood_difference_stderr": 0.5131357048721925, + "pct_stereotype": 0.5652173913043478, + "pct_stereotype_stderr": 0.046429222863564275 + }, + "hendrycksTest-international_law": { + "acc": 0.12396694214876033, + "acc_stderr": 0.030083098716035227, + "acc_norm": 0.2727272727272727, + "acc_norm_stderr": 0.04065578140908705 + }, + "hendrycksTest-anatomy": { + "acc": 0.22962962962962963, + "acc_stderr": 0.036333844140734664, + "acc_norm": 0.2518518518518518, + "acc_norm_stderr": 0.03749850709174023 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2765957446808511, + "acc_stderr": 0.029241883869628827, + "acc_norm": 0.20425531914893616, + "acc_norm_stderr": 0.026355158413349424 + }, + "wsc": { + "acc": 0.5192307692307693, + "acc_stderr": 0.049230010729780505 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.23, + "acc_stderr": 0.042295258468165044, + "acc_norm": 0.22, + "acc_norm_stderr": 0.04163331998932269 + }, + "piqa": { + "acc": 0.5386289445048966, + "acc_stderr": 0.011630956681145914, + "acc_norm": 0.5244831338411317, + "acc_norm_stderr": 0.011651830225709979 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.16, + "acc_stderr": 0.03684529491774709, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036625 + }, + "hendrycksTest-prehistory": { + "acc": 0.27469135802469136, + "acc_stderr": 0.024836057868294688, + "acc_norm": 0.2191358024691358, + "acc_norm_stderr": 0.023016705640262196 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.2369281045751634, + "acc_stderr": 0.017201662169789782, + "acc_norm": 0.2973856209150327, + "acc_norm_stderr": 0.01849259653639695 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.2147239263803681, + "acc_stderr": 0.03226219377286774, + "acc_norm": 0.3128834355828221, + "acc_norm_stderr": 0.036429145782924055 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 4.462456597222222, + "likelihood_difference_stderr": 0.33400887699163057, + "pct_stereotype": 0.33796296296296297, + "pct_stereotype_stderr": 0.03225941352631295 + }, + "hendrycksTest-formal_logic": { + "acc": 0.23015873015873015, + "acc_stderr": 0.03764950879790607, + "acc_norm": 0.25396825396825395, + "acc_norm_stderr": 0.038932596106046755 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.560498046875, + "likelihood_difference_stderr": 0.32984136074752396, + "pct_stereotype": 0.5375, + "pct_stereotype_stderr": 0.02791577963000664 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.189873417721519, + "acc_stderr": 0.025530100460233497, + "acc_norm": 0.23628691983122363, + "acc_norm_stderr": 0.02765215314415925 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.24, + "acc_stderr": 0.042923469599092816, + "acc_norm": 0.21, + "acc_norm_stderr": 0.040936018074033256 + }, + "hendrycksTest-business_ethics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.29, + "acc_norm_stderr": 0.04560480215720684 + }, + "hendrycksTest-world_religions": { + "acc": 0.14035087719298245, + "acc_stderr": 0.0266405825391332, + "acc_norm": 0.21052631578947367, + "acc_norm_stderr": 0.03126781714663179 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 6.849431818181818, + "likelihood_difference_stderr": 2.586994276246196, + "pct_stereotype": 0.36363636363636365, + "pct_stereotype_stderr": 0.15212000482437738 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.20588235294117646, + "acc_stderr": 0.028379449451588667, + "acc_norm": 0.23529411764705882, + "acc_norm_stderr": 0.02977177522814565 + }, + "hendrycksTest-nutrition": { + "acc": 0.17973856209150327, + "acc_stderr": 0.021986032182064148, + "acc_norm": 0.27450980392156865, + "acc_norm_stderr": 0.025553169991826517 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.832756916996048, + "likelihood_difference_stderr": 0.35785767445511346, + "pct_stereotype": 0.3438735177865613, + "pct_stereotype_stderr": 0.029922155720849428 + }, + "arc_easy": { + "acc": 0.2984006734006734, + "acc_stderr": 0.009388855914040428, + "acc_norm": 0.30134680134680136, + "acc_norm_stderr": 0.0094152598793516 + }, + "hendrycksTest-marketing": { + "acc": 0.24358974358974358, + "acc_stderr": 0.0281209665039144, + "acc_norm": 0.25213675213675213, + "acc_norm_stderr": 0.02844796547623101 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.9656434547244093, + "likelihood_difference_stderr": 0.19638996276808376, + "pct_stereotype": 0.468503937007874, + "pct_stereotype_stderr": 0.022161679438492773 + } + }, + "versions": { + "crows_pairs_french": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-high_school_psychology": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-high_school_geography": 0, + "crows_pairs_english": 0, + "logiqa": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-moral_disputes": 0, + "crows_pairs_french_age": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-high_school_physics": 0, + "crows_pairs_french_physical_appearance": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-global_facts": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-management": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-high_school_microeconomics": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-human_aging": 0, + "winogrande": 0, + "crows_pairs_english_disability": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-high_school_biology": 0, + "arc_challenge": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-college_computer_science": 0, + "lambada_openai": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-computer_security": 0, + "crows_pairs_french_race_color": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-miscellaneous": 0, + "sciq": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-public_relations": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-conceptual_physics": 0, + "wsc": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-medical_genetics": 0, + "piqa": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-logical_fallacies": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-formal_logic": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-nutrition": 0, + "crows_pairs_french_nationality": 0, + "arc_easy": 0, + "hendrycksTest-marketing": 0, + "crows_pairs_english_race_color": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step512", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step53000.json b/data/checkpoint_eval/eleutherai_evals/step53000.json new file mode 100644 index 0000000000000000000000000000000000000000..c2a3c4d4937a06693e5e4edb98625952dc86cc2b --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step53000.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-sociology": { + "acc": 0.2736318407960199, + "acc_stderr": 0.03152439186555404, + "acc_norm": 0.3034825870646766, + "acc_norm_stderr": 0.03251006816458617 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.19747899159663865, + "acc_stderr": 0.02585916412205146, + "acc_norm": 0.3025210084033613, + "acc_norm_stderr": 0.02983796238829193 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036623 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2680851063829787, + "acc_stderr": 0.02895734278834235, + "acc_norm": 0.18723404255319148, + "acc_norm_stderr": 0.02550158834188358 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.2037037037037037, + "acc_stderr": 0.027467401804057986, + "acc_norm": 0.22685185185185186, + "acc_norm_stderr": 0.02856165010242227 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.23018867924528302, + "acc_stderr": 0.025907897122408173, + "acc_norm": 0.32452830188679244, + "acc_norm_stderr": 0.028815615713432118 + }, + "piqa": { + "acc": 0.5919477693144722, + "acc_stderr": 0.011466872778651261, + "acc_norm": 0.5979325353645266, + "acc_norm_stderr": 0.01143986712726753 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 5.745738636363637, + "likelihood_difference_stderr": 0.603740965474876, + "pct_stereotype": 0.48484848484848486, + "pct_stereotype_stderr": 0.06198888629778894 + }, + "hendrycksTest-college_medicine": { + "acc": 0.2138728323699422, + "acc_stderr": 0.03126511206173042, + "acc_norm": 0.3063583815028902, + "acc_norm_stderr": 0.03514942551267437 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 4.940384615384615, + "likelihood_difference_stderr": 0.5258513529267634, + "pct_stereotype": 0.6153846153846154, + "pct_stereotype_stderr": 0.06081303192631497 + }, + "hendrycksTest-econometrics": { + "acc": 0.20175438596491227, + "acc_stderr": 0.037752050135836386, + "acc_norm": 0.19298245614035087, + "acc_norm_stderr": 0.037124548537213684 + }, + "hendrycksTest-business_ethics": { + "acc": 0.35, + "acc_stderr": 0.04793724854411018, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542128 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.29533678756476683, + "acc_stderr": 0.032922966391551414, + "acc_norm": 0.27461139896373055, + "acc_norm_stderr": 0.03221024508041154 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 4.6717032967032965, + "likelihood_difference_stderr": 0.35079580322071463, + "pct_stereotype": 0.8021978021978022, + "pct_stereotype_stderr": 0.04198895203196222 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.2, + "acc_stderr": 0.040201512610368466, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 4.72429049744898, + "likelihood_difference_stderr": 0.38514448828446046, + "pct_stereotype": 0.45408163265306123, + "pct_stereotype_stderr": 0.035654431417332814 + }, + "crows_pairs_english": { + "likelihood_difference": 3.67170728980322, + "likelihood_difference_stderr": 0.1032630912208814, + "pct_stereotype": 0.545020870602266, + "pct_stereotype_stderr": 0.012163688705232118 + }, + "crows_pairs_french": { + "likelihood_difference": 5.014772473166368, + "likelihood_difference_stderr": 0.12242859643295022, + "pct_stereotype": 0.43410852713178294, + "pct_stereotype_stderr": 0.012106782103996008 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "acc_norm": 0.26, + "acc_norm_stderr": 0.04408440022768078 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2681992337164751, + "acc_stderr": 0.015842430835269435, + "acc_norm": 0.2515964240102171, + "acc_norm_stderr": 0.015517322365529619 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.2630057803468208, + "acc_stderr": 0.023703099525258155, + "acc_norm": 0.2947976878612717, + "acc_norm_stderr": 0.02454761779480383 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "acc_norm": 0.27, + "acc_norm_stderr": 0.04461960433384741 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.25027932960893856, + "acc_stderr": 0.014487500852850412, + "acc_norm": 0.24692737430167597, + "acc_norm_stderr": 0.014422292204808835 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.2074074074074074, + "acc_stderr": 0.024720713193952148, + "acc_norm": 0.2518518518518518, + "acc_norm_stderr": 0.026466117538959902 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.2647058823529412, + "acc_stderr": 0.03096451792692341, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.03096451792692341 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.173773364485982, + "likelihood_difference_stderr": 0.20666001663696318, + "pct_stereotype": 0.5327102803738317, + "pct_stereotype_stderr": 0.027890972865217984 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.8569878472222223, + "likelihood_difference_stderr": 0.44844825841380226, + "pct_stereotype": 0.5277777777777778, + "pct_stereotype_stderr": 0.05924743948371487 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.18543046357615894, + "acc_stderr": 0.03173284384294287, + "acc_norm": 0.2185430463576159, + "acc_norm_stderr": 0.03374235550425694 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.18518518518518517, + "acc_stderr": 0.03755265865037181, + "acc_norm": 0.37037037037037035, + "acc_norm_stderr": 0.04668408033024931 + }, + "arc_easy": { + "acc": 0.39225589225589225, + "acc_stderr": 0.010018744689650043, + "acc_norm": 0.35858585858585856, + "acc_norm_stderr": 0.009840882301225297 + }, + "hendrycksTest-formal_logic": { + "acc": 0.30952380952380953, + "acc_stderr": 0.04134913018303316, + "acc_norm": 0.29365079365079366, + "acc_norm_stderr": 0.040735243221471255 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.27155963302752295, + "acc_stderr": 0.019069098363191442, + "acc_norm": 0.26605504587155965, + "acc_norm_stderr": 0.018946022322225614 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.355113636363637, + "likelihood_difference_stderr": 1.5602556194869146, + "pct_stereotype": 0.5454545454545454, + "pct_stereotype_stderr": 0.1574591643244434 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.2606060606060606, + "acc_stderr": 0.034277431758165236, + "acc_norm": 0.2787878787878788, + "acc_norm_stderr": 0.035014387062967806 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.9657894736842105, + "likelihood_difference_stderr": 0.2608872260073087, + "pct_stereotype": 0.6473684210526316, + "pct_stereotype_stderr": 0.034754052595820976 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.296551724137931, + "acc_stderr": 0.03806142687309994, + "acc_norm": 0.32413793103448274, + "acc_norm_stderr": 0.03900432069185554 + }, + "hendrycksTest-anatomy": { + "acc": 0.25925925925925924, + "acc_stderr": 0.03785714465066654, + "acc_norm": 0.23703703703703705, + "acc_norm_stderr": 0.03673731683969506 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.307291666666667, + "likelihood_difference_stderr": 0.5547099715245821, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.05933908290969268 + }, + "hendrycksTest-philosophy": { + "acc": 0.2282958199356913, + "acc_stderr": 0.023839303311398215, + "acc_norm": 0.3022508038585209, + "acc_norm_stderr": 0.02608270069539966 + }, + "lambada_openai": { + "ppl": 94.31955728859376, + "ppl_stderr": 3.991574316908998, + "acc": 0.25344459538133124, + "acc_stderr": 0.0060601672763364745 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.28, + "acc_norm_stderr": 0.045126085985421296 + }, + "hendrycksTest-nutrition": { + "acc": 0.25163398692810457, + "acc_stderr": 0.024848018263875192, + "acc_norm": 0.34967320261437906, + "acc_norm_stderr": 0.027305308076274702 + }, + "hendrycksTest-virology": { + "acc": 0.27710843373493976, + "acc_stderr": 0.034843315926805875, + "acc_norm": 0.2891566265060241, + "acc_norm_stderr": 0.03529486801511115 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.0603515625, + "likelihood_difference_stderr": 0.2570312907090984, + "pct_stereotype": 0.5125, + "pct_stereotype_stderr": 0.02798587585995665 + }, + "hendrycksTest-computer_security": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.29, + "acc_norm_stderr": 0.04560480215720683 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2730496453900709, + "acc_stderr": 0.02657786094330786, + "acc_norm": 0.25886524822695034, + "acc_norm_stderr": 0.02612957252718085 + }, + "hendrycksTest-machine_learning": { + "acc": 0.3482142857142857, + "acc_stderr": 0.045218299028335865, + "acc_norm": 0.2767857142857143, + "acc_norm_stderr": 0.042466243366976256 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.5856606791338583, + "likelihood_difference_stderr": 0.18118219123514714, + "pct_stereotype": 0.5118110236220472, + "pct_stereotype_stderr": 0.022199583294816923 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.8061655405405403, + "likelihood_difference_stderr": 0.43453880510820464, + "pct_stereotype": 0.6036036036036037, + "pct_stereotype_stderr": 0.04663848326322447 + }, + "hendrycksTest-management": { + "acc": 0.22330097087378642, + "acc_stderr": 0.04123553189891431, + "acc_norm": 0.3106796116504854, + "acc_norm_stderr": 0.04582124160161551 + }, + "sciq": { + "acc": 0.664, + "acc_stderr": 0.014944140233795028, + "acc_norm": 0.572, + "acc_norm_stderr": 0.01565442624502929 + }, + "hendrycksTest-astronomy": { + "acc": 0.17763157894736842, + "acc_stderr": 0.031103182383123387, + "acc_norm": 0.34868421052631576, + "acc_norm_stderr": 0.03878139888797609 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.23628691983122363, + "acc_stderr": 0.027652153144159294, + "acc_norm": 0.3080168776371308, + "acc_norm_stderr": 0.030052389335605695 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.440149456521739, + "likelihood_difference_stderr": 0.2261395575520835, + "pct_stereotype": 0.3239130434782609, + "pct_stereotype_stderr": 0.021842842500532617 + }, + "hendrycksTest-global_facts": { + "acc": 0.22, + "acc_stderr": 0.04163331998932268, + "acc_norm": 0.21, + "acc_norm_stderr": 0.040936018074033256 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.3053435114503817, + "acc_stderr": 0.040393149787245605, + "acc_norm": 0.2824427480916031, + "acc_norm_stderr": 0.03948406125768361 + }, + "hendrycksTest-prehistory": { + "acc": 0.2993827160493827, + "acc_stderr": 0.02548311560119546, + "acc_norm": 0.23148148148148148, + "acc_norm_stderr": 0.023468429832451145 + }, + "hendrycksTest-college_biology": { + "acc": 0.25, + "acc_stderr": 0.03621034121889507, + "acc_norm": 0.25, + "acc_norm_stderr": 0.03621034121889507 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.878472222222222, + "likelihood_difference_stderr": 0.4858540541132919, + "pct_stereotype": 0.4666666666666667, + "pct_stereotype_stderr": 0.05288198530254015 + }, + "hendrycksTest-marketing": { + "acc": 0.2948717948717949, + "acc_stderr": 0.029872577708891162, + "acc_norm": 0.3162393162393162, + "acc_norm_stderr": 0.030463656747340247 + }, + "hendrycksTest-security_studies": { + "acc": 0.3183673469387755, + "acc_stderr": 0.029822533793982052, + "acc_norm": 0.23265306122448978, + "acc_norm_stderr": 0.02704925791589618 + }, + "hendrycksTest-international_law": { + "acc": 0.2066115702479339, + "acc_stderr": 0.03695980128098823, + "acc_norm": 0.4132231404958678, + "acc_norm_stderr": 0.04495087843548408 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.22486772486772486, + "acc_stderr": 0.021502096078229147, + "acc_norm": 0.20634920634920634, + "acc_norm_stderr": 0.020842290930114676 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.2474747474747475, + "acc_stderr": 0.030746300742124522, + "acc_norm": 0.32323232323232326, + "acc_norm_stderr": 0.033322999210706444 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 4.854619565217392, + "likelihood_difference_stderr": 0.505869033934835, + "pct_stereotype": 0.4956521739130435, + "pct_stereotype_stderr": 0.04682752006203916 + }, + "hendrycksTest-world_religions": { + "acc": 0.2631578947368421, + "acc_stderr": 0.033773102522091945, + "acc_norm": 0.30994152046783624, + "acc_norm_stderr": 0.035469769593931624 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.22699386503067484, + "acc_stderr": 0.032910995786157686, + "acc_norm": 0.2883435582822086, + "acc_norm_stderr": 0.035590395316173425 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.36919466403162, + "likelihood_difference_stderr": 0.3929905019461457, + "pct_stereotype": 0.2964426877470356, + "pct_stereotype_stderr": 0.028768673758013903 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 4.103365384615385, + "likelihood_difference_stderr": 1.0499970465523882, + "pct_stereotype": 0.3076923076923077, + "pct_stereotype_stderr": 0.13323467750529824 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.24193548387096775, + "acc_stderr": 0.024362599693031086, + "acc_norm": 0.3, + "acc_norm_stderr": 0.02606936229533513 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816507, + "acc_norm": 0.35, + "acc_norm_stderr": 0.047937248544110196 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.27, + "acc_stderr": 0.04461960433384739, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "hendrycksTest-professional_law": { + "acc": 0.24837027379400262, + "acc_stderr": 0.01103521259803449, + "acc_norm": 0.27444589308996087, + "acc_norm_stderr": 0.011397043163078154 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.869623655913978, + "likelihood_difference_stderr": 0.5959735406192751, + "pct_stereotype": 0.7849462365591398, + "pct_stereotype_stderr": 0.04283507835554754 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.2565359477124183, + "acc_stderr": 0.017667841612378984, + "acc_norm": 0.25163398692810457, + "acc_norm_stderr": 0.017555818091322256 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.8365162037037037, + "likelihood_difference_stderr": 0.2671010238288838, + "pct_stereotype": 0.4444444444444444, + "pct_stereotype_stderr": 0.03388857118502326 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.3014705882352941, + "acc_stderr": 0.027875982114273168, + "acc_norm": 0.26838235294117646, + "acc_norm_stderr": 0.02691748122437721 + }, + "winogrande": { + "acc": 0.494869771112865, + "acc_stderr": 0.014051745961790516 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.23333333333333334, + "acc_stderr": 0.02144454730156047, + "acc_norm": 0.2717948717948718, + "acc_norm_stderr": 0.02255655101013236 + }, + "hendrycksTest-human_aging": { + "acc": 0.3004484304932735, + "acc_stderr": 0.030769352008229136, + "acc_norm": 0.242152466367713, + "acc_norm_stderr": 0.028751392398694755 + }, + "hendrycksTest-college_physics": { + "acc": 0.19607843137254902, + "acc_stderr": 0.03950581861179962, + "acc_norm": 0.21568627450980393, + "acc_norm_stderr": 0.04092563958237654 + }, + "logiqa": { + "acc": 0.2227342549923195, + "acc_stderr": 0.01632005404616512, + "acc_norm": 0.27956989247311825, + "acc_norm_stderr": 0.017602909186822453 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.19704433497536947, + "acc_stderr": 0.02798672466673622, + "acc_norm": 0.23645320197044334, + "acc_norm_stderr": 0.02989611429173355 + }, + "hendrycksTest-public_relations": { + "acc": 0.2909090909090909, + "acc_stderr": 0.04350271442923243, + "acc_norm": 0.2, + "acc_norm_stderr": 0.038313051408846034 + }, + "arc_challenge": { + "acc": 0.1757679180887372, + "acc_stderr": 0.011122850863120485, + "acc_norm": 0.21331058020477817, + "acc_norm_stderr": 0.011970971742326334 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.652129120879121, + "likelihood_difference_stderr": 0.2944534289937784, + "pct_stereotype": 0.5164835164835165, + "pct_stereotype_stderr": 0.05267597952306975 + } + }, + "versions": { + "hendrycksTest-sociology": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-clinical_knowledge": 0, + "piqa": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-abstract_algebra": 0, + "crows_pairs_french_socioeconomic": 0, + "crows_pairs_english": 0, + "crows_pairs_french": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_french_gender": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-high_school_physics": 0, + "wsc": 0, + "hendrycksTest-jurisprudence": 0, + "arc_easy": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-high_school_psychology": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-high_school_european_history": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-anatomy": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-philosophy": 0, + "lambada_openai": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-virology": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-machine_learning": 0, + "crows_pairs_english_race_color": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-management": 0, + "sciq": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-high_school_world_history": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-college_biology": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-high_school_geography": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-logical_fallacies": 0, + "crows_pairs_french_nationality": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-professional_law": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-professional_psychology": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-professional_medicine": 0, + "winogrande": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-college_physics": 0, + "logiqa": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-public_relations": 0, + "arc_challenge": 0, + "crows_pairs_english_age": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step53000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:5", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step63000.json b/data/checkpoint_eval/eleutherai_evals/step63000.json new file mode 100644 index 0000000000000000000000000000000000000000..bc7b7329689e407463aeea2c661d80fabb525625 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step63000.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-high_school_microeconomics": { + "acc": 0.24789915966386555, + "acc_stderr": 0.028047967224176896, + "acc_norm": 0.3697478991596639, + "acc_norm_stderr": 0.031357095996135904 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.25722543352601157, + "acc_stderr": 0.02353292543104428, + "acc_norm": 0.2861271676300578, + "acc_norm_stderr": 0.02433214677913413 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.23544973544973544, + "acc_stderr": 0.02185150982203172, + "acc_norm": 0.23544973544973544, + "acc_norm_stderr": 0.021851509822031715 + }, + "hendrycksTest-security_studies": { + "acc": 0.3469387755102041, + "acc_stderr": 0.0304725260267265, + "acc_norm": 0.23673469387755103, + "acc_norm_stderr": 0.02721283588407315 + }, + "hendrycksTest-philosophy": { + "acc": 0.18971061093247588, + "acc_stderr": 0.022268196258783218, + "acc_norm": 0.2829581993569132, + "acc_norm_stderr": 0.025583062489984838 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.16, + "acc_stderr": 0.03684529491774709, + "acc_norm": 0.24, + "acc_norm_stderr": 0.042923469599092816 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 4.8039772727272725, + "likelihood_difference_stderr": 1.5170731185765034, + "pct_stereotype": 0.5454545454545454, + "pct_stereotype_stderr": 0.1574591643244434 + }, + "hendrycksTest-human_aging": { + "acc": 0.3004484304932735, + "acc_stderr": 0.03076935200822914, + "acc_norm": 0.21524663677130046, + "acc_norm_stderr": 0.027584066602208263 + }, + "lambada_openai": { + "ppl": 101.68439461161867, + "ppl_stderr": 4.289464289805073, + "acc": 0.23287405394915583, + "acc_stderr": 0.00588851737109305 + }, + "crows_pairs_english": { + "likelihood_difference": 3.5585494931425163, + "likelihood_difference_stderr": 0.10448474706694104, + "pct_stereotype": 0.545020870602266, + "pct_stereotype_stderr": 0.012163688705232118 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.28, + "acc_stderr": 0.04512608598542127, + "acc_norm": 0.35, + "acc_norm_stderr": 0.0479372485441102 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.626591435185185, + "likelihood_difference_stderr": 0.25256110110560454, + "pct_stereotype": 0.4305555555555556, + "pct_stereotype_stderr": 0.03376922151252336 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.23, + "acc_stderr": 0.04229525846816508, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.325545171339564, + "likelihood_difference_stderr": 0.21316376809344068, + "pct_stereotype": 0.5233644859813084, + "pct_stereotype_stderr": 0.02792031634820498 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.2647058823529412, + "acc_stderr": 0.026799562024887674, + "acc_norm": 0.25, + "acc_norm_stderr": 0.026303648393696036 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.393002717391305, + "likelihood_difference_stderr": 0.22538878372864266, + "pct_stereotype": 0.40652173913043477, + "pct_stereotype_stderr": 0.022926510173270086 + }, + "piqa": { + "acc": 0.5930359085963003, + "acc_stderr": 0.011462093919190166, + "acc_norm": 0.5990206746463548, + "acc_norm_stderr": 0.011434766962108316 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.674395161290323, + "likelihood_difference_stderr": 0.5663537493866246, + "pct_stereotype": 0.8064516129032258, + "pct_stereotype_stderr": 0.041189832133487855 + }, + "hendrycksTest-professional_law": { + "acc": 0.2692307692307692, + "acc_stderr": 0.01132873440314032, + "acc_norm": 0.27835723598435463, + "acc_norm_stderr": 0.011446990197380989 + }, + "hendrycksTest-public_relations": { + "acc": 0.24545454545454545, + "acc_stderr": 0.041220665028782834, + "acc_norm": 0.19090909090909092, + "acc_norm_stderr": 0.03764425585984924 + }, + "hendrycksTest-anatomy": { + "acc": 0.21481481481481482, + "acc_stderr": 0.035478541985608236, + "acc_norm": 0.22962962962962963, + "acc_norm_stderr": 0.03633384414073462 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 4.721580038265306, + "likelihood_difference_stderr": 0.3810892591687784, + "pct_stereotype": 0.41836734693877553, + "pct_stereotype_stderr": 0.03532530943876561 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.26993865030674846, + "acc_stderr": 0.03487825168497892, + "acc_norm": 0.294478527607362, + "acc_norm_stderr": 0.03581165790474082 + }, + "hendrycksTest-machine_learning": { + "acc": 0.2767857142857143, + "acc_stderr": 0.042466243366976256, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04109974682633932 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2765957446808511, + "acc_stderr": 0.02668456434046101, + "acc_norm": 0.26595744680851063, + "acc_norm_stderr": 0.026358065698880582 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2620689655172414, + "acc_stderr": 0.036646663372252565, + "acc_norm": 0.31724137931034485, + "acc_norm_stderr": 0.038783523721386215 + }, + "hendrycksTest-management": { + "acc": 0.18446601941747573, + "acc_stderr": 0.03840423627288276, + "acc_norm": 0.21359223300970873, + "acc_norm_stderr": 0.04058042015646034 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.32061068702290074, + "acc_stderr": 0.04093329229834278, + "acc_norm": 0.2748091603053435, + "acc_norm_stderr": 0.03915345408847835 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.8314144736842106, + "likelihood_difference_stderr": 0.2638399370112327, + "pct_stereotype": 0.6105263157894737, + "pct_stereotype_stderr": 0.035469931637371596 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.534855769230769, + "likelihood_difference_stderr": 0.2981689993648672, + "pct_stereotype": 0.5054945054945055, + "pct_stereotype_stderr": 0.05270144531112881 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.341346153846154, + "likelihood_difference_stderr": 0.6140137588967796, + "pct_stereotype": 0.6307692307692307, + "pct_stereotype_stderr": 0.06032456592830047 + }, + "hendrycksTest-astronomy": { + "acc": 0.19078947368421054, + "acc_stderr": 0.031975658210325, + "acc_norm": 0.3684210526315789, + "acc_norm_stderr": 0.03925523381052932 + }, + "hendrycksTest-marketing": { + "acc": 0.2606837606837607, + "acc_stderr": 0.028760348956523414, + "acc_norm": 0.2948717948717949, + "acc_norm_stderr": 0.02987257770889117 + }, + "hendrycksTest-nutrition": { + "acc": 0.2581699346405229, + "acc_stderr": 0.025058503316958157, + "acc_norm": 0.3300653594771242, + "acc_norm_stderr": 0.02692565465361569 + }, + "hendrycksTest-college_medicine": { + "acc": 0.24855491329479767, + "acc_stderr": 0.03295304696818318, + "acc_norm": 0.3179190751445087, + "acc_norm_stderr": 0.03550683989165581 + }, + "hendrycksTest-international_law": { + "acc": 0.19008264462809918, + "acc_stderr": 0.03581796951709282, + "acc_norm": 0.38016528925619836, + "acc_norm_stderr": 0.04431324501968431 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 4.681521739130435, + "likelihood_difference_stderr": 0.49794984189910335, + "pct_stereotype": 0.5826086956521739, + "pct_stereotype_stderr": 0.04618572379512261 + }, + "hendrycksTest-prehistory": { + "acc": 0.31790123456790126, + "acc_stderr": 0.025910063528240868, + "acc_norm": 0.23765432098765432, + "acc_norm_stderr": 0.02368359183700855 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.2434640522875817, + "acc_stderr": 0.017362473762146634, + "acc_norm": 0.272875816993464, + "acc_norm_stderr": 0.018020474148393577 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.22641509433962265, + "acc_stderr": 0.025757559893106737, + "acc_norm": 0.3283018867924528, + "acc_norm_stderr": 0.028901593612411784 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 6.927197802197802, + "likelihood_difference_stderr": 0.492595571005076, + "pct_stereotype": 0.7912087912087912, + "pct_stereotype_stderr": 0.04284305206509431 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.29541284403669726, + "acc_stderr": 0.019560619182976, + "acc_norm": 0.26788990825688075, + "acc_norm_stderr": 0.018987462257978652 + }, + "arc_easy": { + "acc": 0.4090909090909091, + "acc_stderr": 0.01008877515261578, + "acc_norm": 0.3720538720538721, + "acc_norm_stderr": 0.00991818719309646 + }, + "hendrycksTest-global_facts": { + "acc": 0.29, + "acc_stderr": 0.045604802157206845, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542127 + }, + "hendrycksTest-business_ethics": { + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "acc_norm": 0.27, + "acc_norm_stderr": 0.0446196043338474 + }, + "crows_pairs_french": { + "likelihood_difference": 5.22263994484198, + "likelihood_difference_stderr": 0.12633351172363225, + "pct_stereotype": 0.4531902206320811, + "pct_stereotype_stderr": 0.012159658951661536 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542127 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.8509114583333335, + "likelihood_difference_stderr": 0.4282178435765489, + "pct_stereotype": 0.6388888888888888, + "pct_stereotype_stderr": 0.05700381461700859 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.573616600790514, + "likelihood_difference_stderr": 0.40025626909052686, + "pct_stereotype": 0.2845849802371542, + "pct_stereotype_stderr": 0.028423970522085215 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.33, + "acc_norm_stderr": 0.04725815626252606 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.27330779054916987, + "acc_stderr": 0.015936681062628556, + "acc_norm": 0.24393358876117496, + "acc_norm_stderr": 0.015357212665829475 + }, + "hendrycksTest-world_religions": { + "acc": 0.26900584795321636, + "acc_stderr": 0.0340105262010409, + "acc_norm": 0.32748538011695905, + "acc_norm_stderr": 0.035993357714560276 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.5273085585585586, + "likelihood_difference_stderr": 0.42554000873855896, + "pct_stereotype": 0.6396396396396397, + "pct_stereotype_stderr": 0.04577621167070314 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.29, + "acc_stderr": 0.04560480215720683, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.388257575757576, + "likelihood_difference_stderr": 0.6176380444815222, + "pct_stereotype": 0.4393939393939394, + "pct_stereotype_stderr": 0.06156009014560979 + }, + "hendrycksTest-sociology": { + "acc": 0.27860696517412936, + "acc_stderr": 0.031700561834973086, + "acc_norm": 0.31343283582089554, + "acc_norm_stderr": 0.032801882053486435 + }, + "logiqa": { + "acc": 0.22427035330261136, + "acc_stderr": 0.016360043348265515, + "acc_norm": 0.26881720430107525, + "acc_norm_stderr": 0.01738940946371263 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.17407407407407408, + "acc_stderr": 0.02311859603355185, + "acc_norm": 0.26666666666666666, + "acc_norm_stderr": 0.026962424325073838 + }, + "hendrycksTest-college_biology": { + "acc": 0.3055555555555556, + "acc_stderr": 0.03852084696008534, + "acc_norm": 0.2847222222222222, + "acc_norm_stderr": 0.03773809990686934 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.28431372549019607, + "acc_stderr": 0.031660096793998116, + "acc_norm": 0.2696078431372549, + "acc_norm_stderr": 0.031145570659486782 + }, + "hendrycksTest-college_physics": { + "acc": 0.17647058823529413, + "acc_stderr": 0.03793281185307809, + "acc_norm": 0.24509803921568626, + "acc_norm_stderr": 0.04280105837364396 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.5295890748031495, + "likelihood_difference_stderr": 0.19219930990870296, + "pct_stereotype": 0.484251968503937, + "pct_stereotype_stderr": 0.02219476276265932 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.20833333333333334, + "acc_stderr": 0.02769691071309394, + "acc_norm": 0.2638888888888889, + "acc_norm_stderr": 0.03005820270430985 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.15894039735099338, + "acc_stderr": 0.02985278852870104, + "acc_norm": 0.2185430463576159, + "acc_norm_stderr": 0.03374235550425694 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.2037037037037037, + "acc_stderr": 0.03893542518824847, + "acc_norm": 0.37037037037037035, + "acc_norm_stderr": 0.04668408033024931 + }, + "arc_challenge": { + "acc": 0.18003412969283278, + "acc_stderr": 0.011227856729050054, + "acc_norm": 0.20733788395904437, + "acc_norm_stderr": 0.011846905782971363 + }, + "winogrande": { + "acc": 0.4996053670086819, + "acc_stderr": 0.01405248130604952 + }, + "hendrycksTest-econometrics": { + "acc": 0.21052631578947367, + "acc_stderr": 0.038351539543994194, + "acc_norm": 0.20175438596491227, + "acc_norm_stderr": 0.037752050135836386 + }, + "hendrycksTest-virology": { + "acc": 0.24096385542168675, + "acc_stderr": 0.033293941190735296, + "acc_norm": 0.2289156626506024, + "acc_norm_stderr": 0.03270745277352477 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.25161290322580643, + "acc_stderr": 0.02468597928623996, + "acc_norm": 0.3419354838709677, + "acc_norm_stderr": 0.026985289576552732 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 2.903515625, + "likelihood_difference_stderr": 0.25564415610007174, + "pct_stereotype": 0.54375, + "pct_stereotype_stderr": 0.027887252708654657 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.650173611111111, + "likelihood_difference_stderr": 0.6364202127315652, + "pct_stereotype": 0.4444444444444444, + "pct_stereotype_stderr": 0.05897165471491952 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.25384615384615383, + "acc_stderr": 0.022066054378726257, + "acc_norm": 0.2743589743589744, + "acc_norm_stderr": 0.02262276576749322 + }, + "hendrycksTest-formal_logic": { + "acc": 0.30952380952380953, + "acc_stderr": 0.04134913018303317, + "acc_norm": 0.2698412698412698, + "acc_norm_stderr": 0.03970158273235173 + }, + "sciq": { + "acc": 0.681, + "acc_stderr": 0.014746404865473484, + "acc_norm": 0.616, + "acc_norm_stderr": 0.01538768276189707 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 4.677884615384615, + "likelihood_difference_stderr": 0.9490566058977856, + "pct_stereotype": 0.46153846153846156, + "pct_stereotype_stderr": 0.14390989949130545 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2680851063829787, + "acc_stderr": 0.02895734278834235, + "acc_norm": 0.20425531914893616, + "acc_norm_stderr": 0.026355158413349424 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.2538860103626943, + "acc_stderr": 0.03141024780565318, + "acc_norm": 0.24352331606217617, + "acc_norm_stderr": 0.03097543638684542 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.21674876847290642, + "acc_stderr": 0.028990331252516235, + "acc_norm": 0.2857142857142857, + "acc_norm_stderr": 0.031785297106427496 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.21818181818181817, + "acc_stderr": 0.03225078108306289, + "acc_norm": 0.24848484848484848, + "acc_norm_stderr": 0.03374402644139403 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.2676767676767677, + "acc_stderr": 0.03154449888270286, + "acc_norm": 0.3282828282828283, + "acc_norm_stderr": 0.03345678422756775 + }, + "hendrycksTest-computer_security": { + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "acc_norm": 0.26, + "acc_norm_stderr": 0.04408440022768081 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.994791666666667, + "likelihood_difference_stderr": 0.5220305178059566, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.052999894000318 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.23798882681564246, + "acc_norm_stderr": 0.014242630070574915 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.23628691983122363, + "acc_stderr": 0.027652153144159277, + "acc_norm": 0.3206751054852321, + "acc_norm_stderr": 0.03038193194999042 + } + }, + "versions": { + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-college_mathematics": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-human_aging": 0, + "lambada_openai": 0, + "crows_pairs_english": 0, + "hendrycksTest-college_chemistry": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-abstract_algebra": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-professional_medicine": 0, + "crows_pairs_french_race_color": 0, + "piqa": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-anatomy": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-management": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_english_socioeconomic": 0, + "crows_pairs_english_age": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-international_law": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-high_school_psychology": 0, + "arc_easy": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-business_ethics": 0, + "crows_pairs_french": 0, + "hendrycksTest-college_computer_science": 0, + "crows_pairs_english_physical_appearance": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-us_foreign_policy": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-sociology": 0, + "logiqa": 0, + "wsc": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-college_physics": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-jurisprudence": 0, + "arc_challenge": 0, + "winogrande": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-high_school_biology": 0, + "crows_pairs_english_gender": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-formal_logic": 0, + "sciq": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-computer_security": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-high_school_world_history": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step63000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:6", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step64.json b/data/checkpoint_eval/eleutherai_evals/step64.json new file mode 100644 index 0000000000000000000000000000000000000000..8cecb35b293dd71cab554326aaa97395cdaca409 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step64.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-elementary_mathematics": { + "acc": 0.21693121693121692, + "acc_stderr": 0.021227082449445045, + "acc_norm": 0.21164021164021163, + "acc_norm_stderr": 0.02103733150526289 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.1729957805907173, + "acc_stderr": 0.024621562866768427, + "acc_norm": 0.25316455696202533, + "acc_norm_stderr": 0.028304657943035296 + }, + "winogrande": { + "acc": 0.4972375690607735, + "acc_stderr": 0.014052271211616445 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.21794871794871795, + "acc_stderr": 0.020932445774463175, + "acc_norm": 0.26153846153846155, + "acc_norm_stderr": 0.02228214120420442 + }, + "hendrycksTest-machine_learning": { + "acc": 0.24107142857142858, + "acc_stderr": 0.04059867246952685, + "acc_norm": 0.2857142857142857, + "acc_norm_stderr": 0.04287858751340456 + }, + "hendrycksTest-college_physics": { + "acc": 0.22549019607843138, + "acc_stderr": 0.041583075330832865, + "acc_norm": 0.23529411764705882, + "acc_norm_stderr": 0.04220773659171452 + }, + "hendrycksTest-anatomy": { + "acc": 0.23703703703703705, + "acc_stderr": 0.03673731683969506, + "acc_norm": 0.28888888888888886, + "acc_norm_stderr": 0.0391545063041425 + }, + "hendrycksTest-management": { + "acc": 0.23300970873786409, + "acc_stderr": 0.04185832598928315, + "acc_norm": 0.23300970873786409, + "acc_norm_stderr": 0.04185832598928315 + }, + "hendrycksTest-college_medicine": { + "acc": 0.21965317919075145, + "acc_stderr": 0.031568093627031744, + "acc_norm": 0.2658959537572254, + "acc_norm_stderr": 0.03368762932259431 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.28936170212765955, + "acc_stderr": 0.029644006577009618, + "acc_norm": 0.24680851063829787, + "acc_norm_stderr": 0.02818544130123408 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.2222222222222222, + "acc_stderr": 0.01681902837573638, + "acc_norm": 0.24836601307189543, + "acc_norm_stderr": 0.017479487001364764 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.21717171717171718, + "acc_stderr": 0.029376616484945637, + "acc_norm": 0.26262626262626265, + "acc_norm_stderr": 0.031353050095330855 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 5.821180555555555, + "likelihood_difference_stderr": 0.7528994326490429, + "pct_stereotype": 0.5416666666666666, + "pct_stereotype_stderr": 0.05913268547421811 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 7.460576923076923, + "likelihood_difference_stderr": 1.1593364221878786, + "pct_stereotype": 0.5076923076923077, + "pct_stereotype_stderr": 0.062492603112584276 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.17177914110429449, + "acc_stderr": 0.02963471727237102, + "acc_norm": 0.27607361963190186, + "acc_norm_stderr": 0.0351238528370505 + }, + "hendrycksTest-security_studies": { + "acc": 0.3183673469387755, + "acc_stderr": 0.02982253379398205, + "acc_norm": 0.21224489795918366, + "acc_norm_stderr": 0.026176967197866767 + }, + "hendrycksTest-virology": { + "acc": 0.16265060240963855, + "acc_stderr": 0.028730237892613798, + "acc_norm": 0.21084337349397592, + "acc_norm_stderr": 0.031755547866299215 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.22, + "acc_stderr": 0.04163331998932269, + "acc_norm": 0.22, + "acc_norm_stderr": 0.0416333199893227 + }, + "hendrycksTest-econometrics": { + "acc": 0.22807017543859648, + "acc_stderr": 0.03947152782669415, + "acc_norm": 0.32456140350877194, + "acc_norm_stderr": 0.04404556157374768 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.21243523316062177, + "acc_stderr": 0.029519282616817254, + "acc_norm": 0.24870466321243523, + "acc_norm_stderr": 0.03119584087770029 + }, + "logiqa": { + "acc": 0.20890937019969277, + "acc_stderr": 0.015945399396423896, + "acc_norm": 0.24270353302611367, + "acc_norm_stderr": 0.016815676206479523 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2366412213740458, + "acc_stderr": 0.03727673575596919, + "acc_norm": 0.25190839694656486, + "acc_norm_stderr": 0.038073871163060866 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 6.8173076923076925, + "likelihood_difference_stderr": 1.7019864539843879, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.14390989949130545 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 14.00189393939394, + "likelihood_difference_stderr": 1.4054078154752692, + "pct_stereotype": 0.36363636363636365, + "pct_stereotype_stderr": 0.05966637484671757 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.23486238532110093, + "acc_stderr": 0.01817511051034359, + "acc_norm": 0.24954128440366974, + "acc_norm_stderr": 0.018553897629501624 + }, + "sciq": { + "acc": 0.199, + "acc_stderr": 0.012631649083099186, + "acc_norm": 0.218, + "acc_norm_stderr": 0.013063179040595282 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 11.76251594387755, + "likelihood_difference_stderr": 0.7977323948668976, + "pct_stereotype": 0.3673469387755102, + "pct_stereotype_stderr": 0.03452261728704165 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 11.039673913043478, + "likelihood_difference_stderr": 0.8746581584196995, + "pct_stereotype": 0.6608695652173913, + "pct_stereotype_stderr": 0.04433930011819816 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.18518518518518517, + "acc_stderr": 0.03755265865037181, + "acc_norm": 0.24074074074074073, + "acc_norm_stderr": 0.041331194402438376 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 7.126947040498442, + "likelihood_difference_stderr": 0.41934707639834146, + "pct_stereotype": 0.48598130841121495, + "pct_stereotype_stderr": 0.027939861549302374 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.13, + "acc_stderr": 0.03379976689896309, + "acc_norm": 0.21, + "acc_norm_stderr": 0.040936018074033256 + }, + "arc_challenge": { + "acc": 0.20819112627986347, + "acc_stderr": 0.01186486611844807, + "acc_norm": 0.24488054607508533, + "acc_norm_stderr": 0.012566273985131356 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.1660377358490566, + "acc_stderr": 0.022902064724569966, + "acc_norm": 0.3018867924528302, + "acc_norm_stderr": 0.028254200344438655 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.184659090909091, + "likelihood_difference_stderr": 2.7270102264769593, + "pct_stereotype": 0.6363636363636364, + "pct_stereotype_stderr": 0.15212000482437738 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 5.49078947368421, + "likelihood_difference_stderr": 0.48194076993788276, + "pct_stereotype": 0.6157894736842106, + "pct_stereotype_stderr": 0.03538097998767891 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.28, + "acc_stderr": 0.04512608598542128, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 5.203678641732283, + "likelihood_difference_stderr": 0.3157259260594444, + "pct_stereotype": 0.33267716535433073, + "pct_stereotype_stderr": 0.02092548388333584 + }, + "hendrycksTest-human_aging": { + "acc": 0.2645739910313901, + "acc_stderr": 0.029605103217038315, + "acc_norm": 0.2825112107623318, + "acc_norm_stderr": 0.03021683101150877 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 8.90736166007905, + "likelihood_difference_stderr": 0.48994737960034646, + "pct_stereotype": 0.5177865612648221, + "pct_stereotype_stderr": 0.03147710419094347 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.2222222222222222, + "acc_stderr": 0.028353212866863445, + "acc_norm": 0.2824074074074074, + "acc_norm_stderr": 0.030701372111510937 + }, + "hendrycksTest-philosophy": { + "acc": 0.2508038585209003, + "acc_stderr": 0.024619771956697168, + "acc_norm": 0.2765273311897106, + "acc_norm_stderr": 0.02540383297817961 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.24265644955300128, + "acc_stderr": 0.01532988894089986, + "acc_norm": 0.26053639846743293, + "acc_norm_stderr": 0.01569600856380707 + }, + "hendrycksTest-professional_law": { + "acc": 0.2242503259452412, + "acc_stderr": 0.010652615824906163, + "acc_norm": 0.2529335071707953, + "acc_norm_stderr": 0.011102268713839989 + }, + "hendrycksTest-astronomy": { + "acc": 0.19736842105263158, + "acc_stderr": 0.03238981601699397, + "acc_norm": 0.24342105263157895, + "acc_norm_stderr": 0.034923496688842384 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 5.985243055555555, + "likelihood_difference_stderr": 0.48650219522921606, + "pct_stereotype": 0.3333333333333333, + "pct_stereotype_stderr": 0.03214952147802749 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.27, + "acc_norm_stderr": 0.0446196043338474 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.22660098522167488, + "acc_stderr": 0.029454863835292968, + "acc_norm": 0.21674876847290642, + "acc_norm_stderr": 0.028990331252516235 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.17575757575757575, + "acc_stderr": 0.02972094300622445, + "acc_norm": 0.23030303030303031, + "acc_norm_stderr": 0.03287666758603488 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.19117647058823528, + "acc_stderr": 0.027599174300640766, + "acc_norm": 0.2696078431372549, + "acc_norm_stderr": 0.031145570659486782 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.23, + "acc_stderr": 0.042295258468165044, + "acc_norm": 0.22, + "acc_norm_stderr": 0.04163331998932269 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 8.887228260869565, + "likelihood_difference_stderr": 0.3298429589043161, + "pct_stereotype": 0.717391304347826, + "pct_stereotype_stderr": 0.02101669741793868 + }, + "crows_pairs_french": { + "likelihood_difference": 9.346535852713178, + "likelihood_difference_stderr": 0.21437404785240546, + "pct_stereotype": 0.5742397137745975, + "pct_stereotype_stderr": 0.012077920863042001 + }, + "hendrycksTest-world_religions": { + "acc": 0.17543859649122806, + "acc_stderr": 0.02917088550072768, + "acc_norm": 0.2222222222222222, + "acc_norm_stderr": 0.03188578017686399 + }, + "hendrycksTest-global_facts": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.22, + "acc_norm_stderr": 0.04163331998932269 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.2, + "acc_stderr": 0.04020151261036843, + "acc_norm": 0.15, + "acc_norm_stderr": 0.035887028128263734 + }, + "hendrycksTest-business_ethics": { + "acc": 0.32, + "acc_stderr": 0.04688261722621504, + "acc_norm": 0.27, + "acc_norm_stderr": 0.0446196043338474 + }, + "hendrycksTest-international_law": { + "acc": 0.09917355371900827, + "acc_stderr": 0.027285246312758957, + "acc_norm": 0.2809917355371901, + "acc_norm_stderr": 0.04103203830514512 + }, + "hendrycksTest-college_biology": { + "acc": 0.2013888888888889, + "acc_stderr": 0.03353647469713839, + "acc_norm": 0.2361111111111111, + "acc_norm_stderr": 0.03551446610810826 + }, + "wsc": { + "acc": 0.375, + "acc_stderr": 0.04770204856076104 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.18067226890756302, + "acc_stderr": 0.024991964966600753, + "acc_norm": 0.2857142857142857, + "acc_norm_stderr": 0.029344572500634342 + }, + "hendrycksTest-prehistory": { + "acc": 0.2623456790123457, + "acc_stderr": 0.02447722285613512, + "acc_norm": 0.23765432098765432, + "acc_norm_stderr": 0.023683591837008557 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2695035460992908, + "acc_stderr": 0.02646903681859063, + "acc_norm": 0.2801418439716312, + "acc_norm_stderr": 0.02678917235114024 + }, + "piqa": { + "acc": 0.5402611534276387, + "acc_stderr": 0.011627942981817168, + "acc_norm": 0.5195865070729053, + "acc_norm_stderr": 0.011656869979288454 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "hendrycksTest-computer_security": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.29, + "acc_norm_stderr": 0.045604802157206845 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.19032258064516128, + "acc_stderr": 0.02233170761182307, + "acc_norm": 0.24516129032258063, + "acc_norm_stderr": 0.02447224384089552 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.1814814814814815, + "acc_stderr": 0.023499264669407306, + "acc_norm": 0.27037037037037037, + "acc_norm_stderr": 0.027080372815145658 + }, + "crows_pairs_english": { + "likelihood_difference": 5.231738223017293, + "likelihood_difference_stderr": 0.1788777548247783, + "pct_stereotype": 0.43530113297555156, + "pct_stereotype_stderr": 0.012110619233278561 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.415994623655914, + "likelihood_difference_stderr": 0.7173989239033504, + "pct_stereotype": 0.45161290322580644, + "pct_stereotype_stderr": 0.051883930752016603 + }, + "hendrycksTest-nutrition": { + "acc": 0.20261437908496732, + "acc_stderr": 0.023015446877985693, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.02526169121972948 + }, + "hendrycksTest-sociology": { + "acc": 0.2885572139303483, + "acc_stderr": 0.0320384104021332, + "acc_norm": 0.31343283582089554, + "acc_norm_stderr": 0.03280188205348642 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.22, + "acc_stderr": 0.041633319989322695, + "acc_norm": 0.36, + "acc_norm_stderr": 0.04824181513244218 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.3361950549450547, + "likelihood_difference_stderr": 0.5594307162046473, + "pct_stereotype": 0.45054945054945056, + "pct_stereotype_stderr": 0.052446231001012276 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.1986754966887417, + "acc_stderr": 0.032578473844367774, + "acc_norm": 0.2781456953642384, + "acc_norm_stderr": 0.036586032627637426 + }, + "hendrycksTest-public_relations": { + "acc": 0.3, + "acc_stderr": 0.04389311454644287, + "acc_norm": 0.17272727272727273, + "acc_norm_stderr": 0.03620691833929219 + }, + "arc_easy": { + "acc": 0.2706228956228956, + "acc_stderr": 0.009116466166403832, + "acc_norm": 0.26515151515151514, + "acc_norm_stderr": 0.009057621139172618 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.21676300578034682, + "acc_stderr": 0.022183477668412853, + "acc_norm": 0.20520231213872833, + "acc_norm_stderr": 0.021742519835276277 + }, + "hendrycksTest-marketing": { + "acc": 0.19230769230769232, + "acc_stderr": 0.025819233256483713, + "acc_norm": 0.24358974358974358, + "acc_norm_stderr": 0.0281209665039144 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.21379310344827587, + "acc_stderr": 0.03416520447747549, + "acc_norm": 0.20689655172413793, + "acc_norm_stderr": 0.03375672449560554 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 9.36111111111111, + "likelihood_difference_stderr": 1.299896180819777, + "pct_stereotype": 0.5833333333333334, + "pct_stereotype_stderr": 0.05850912479161746 + }, + "hendrycksTest-formal_logic": { + "acc": 0.29365079365079366, + "acc_stderr": 0.040735243221471276, + "acc_norm": 0.2777777777777778, + "acc_norm_stderr": 0.04006168083848876 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 5.315596846846847, + "likelihood_difference_stderr": 0.6585436977642254, + "pct_stereotype": 0.40540540540540543, + "pct_stereotype_stderr": 0.046812183988348 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.2426470588235294, + "acc_stderr": 0.026040662474201264, + "acc_norm": 0.28308823529411764, + "acc_norm_stderr": 0.02736586113151381 + }, + "lambada_openai": { + "ppl": 2347965.083490206, + "ppl_stderr": 208687.88666648002, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 4.4865234375, + "likelihood_difference_stderr": 0.46807997363511794, + "pct_stereotype": 0.515625, + "pct_stereotype_stderr": 0.027980952958187033 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 14.45467032967033, + "likelihood_difference_stderr": 1.0156884564726079, + "pct_stereotype": 0.8021978021978022, + "pct_stereotype_stderr": 0.04198895203196222 + }, + "crows_pairs_french_age": { + "likelihood_difference": 5.195138888888889, + "likelihood_difference_stderr": 0.6908977353059731, + "pct_stereotype": 0.5777777777777777, + "pct_stereotype_stderr": 0.05235473399540658 + } + }, + "versions": { + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-high_school_world_history": 0, + "winogrande": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-management": 0, + "hendrycksTest-college_medicine": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-high_school_geography": 0, + "crows_pairs_english_physical_appearance": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "logiqa": 0, + "hendrycksTest-human_sexuality": 0, + "crows_pairs_french_autre": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-high_school_psychology": 0, + "sciq": 0, + "crows_pairs_french_socioeconomic": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-jurisprudence": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-college_mathematics": 0, + "arc_challenge": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_english_autre": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-medical_genetics": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-human_aging": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-astronomy": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-college_computer_science": 0, + "crows_pairs_french_race_color": 0, + "crows_pairs_french": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-college_biology": 0, + "wsc": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-professional_accounting": 0, + "piqa": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-high_school_mathematics": 0, + "crows_pairs_english": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-high_school_computer_science": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-public_relations": 0, + "arc_easy": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-electrical_engineering": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-formal_logic": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-professional_medicine": 0, + "lambada_openai": 0, + "crows_pairs_english_gender": 0, + "crows_pairs_french_sexual_orientation": 0, + "crows_pairs_french_age": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step64", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step73000.json b/data/checkpoint_eval/eleutherai_evals/step73000.json new file mode 100644 index 0000000000000000000000000000000000000000..c97bb3c8c0ef9013472cbb8dfcb015942b6fb4e9 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step73000.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.22564102564102564, + "acc_stderr": 0.021193632525148522, + "acc_norm": 0.2743589743589744, + "acc_norm_stderr": 0.022622765767493228 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.23949579831932774, + "acc_stderr": 0.027722065493361252, + "acc_norm": 0.3445378151260504, + "acc_norm_stderr": 0.030868682604121633 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.09404296875, + "likelihood_difference_stderr": 0.2767365631867781, + "pct_stereotype": 0.571875, + "pct_stereotype_stderr": 0.02770387433578863 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 4.71875, + "likelihood_difference_stderr": 1.3094458513050473, + "pct_stereotype": 0.5454545454545454, + "pct_stereotype_stderr": 0.1574591643244434 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.27889908256880735, + "acc_stderr": 0.01922746887646351, + "acc_norm": 0.25321100917431194, + "acc_norm_stderr": 0.018644073041375046 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.22, + "acc_norm_stderr": 0.04163331998932269 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.805434782608696, + "likelihood_difference_stderr": 0.2500947844192814, + "pct_stereotype": 0.29347826086956524, + "pct_stereotype_stderr": 0.02125418176290879 + }, + "winogrande": { + "acc": 0.5146014206787688, + "acc_stderr": 0.014046492383275835 + }, + "hendrycksTest-philosophy": { + "acc": 0.19292604501607716, + "acc_stderr": 0.022411516780911366, + "acc_norm": 0.28938906752411575, + "acc_norm_stderr": 0.025755865922632935 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.6137152777777777, + "likelihood_difference_stderr": 0.40738652490063665, + "pct_stereotype": 0.6527777777777778, + "pct_stereotype_stderr": 0.056501146768529645 + }, + "piqa": { + "acc": 0.5984766050054406, + "acc_stderr": 0.011437324373397846, + "acc_norm": 0.5984766050054406, + "acc_norm_stderr": 0.011437324373397843 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.25316455696202533, + "acc_stderr": 0.02830465794303529, + "acc_norm": 0.2742616033755274, + "acc_norm_stderr": 0.029041333510598035 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.23448275862068965, + "acc_stderr": 0.035306258743465914, + "acc_norm": 0.30344827586206896, + "acc_norm_stderr": 0.038312260488503336 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "hendrycksTest-formal_logic": { + "acc": 0.3253968253968254, + "acc_stderr": 0.041905964388711366, + "acc_norm": 0.30158730158730157, + "acc_norm_stderr": 0.041049472699033945 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.2514450867052023, + "acc_stderr": 0.02335736578587404, + "acc_norm": 0.28901734104046245, + "acc_norm_stderr": 0.024405173935783234 + }, + "crows_pairs_english": { + "likelihood_difference": 3.558437686344663, + "likelihood_difference_stderr": 0.1036189376756294, + "pct_stereotype": 0.5420393559928444, + "pct_stereotype_stderr": 0.012170053344890804 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.88508064516129, + "likelihood_difference_stderr": 0.531511439742182, + "pct_stereotype": 0.7956989247311828, + "pct_stereotype_stderr": 0.04203545939892302 + }, + "hendrycksTest-public_relations": { + "acc": 0.3, + "acc_stderr": 0.04389311454644286, + "acc_norm": 0.19090909090909092, + "acc_norm_stderr": 0.03764425585984924 + }, + "hendrycksTest-prehistory": { + "acc": 0.2962962962962963, + "acc_stderr": 0.025407197798890162, + "acc_norm": 0.24074074074074073, + "acc_norm_stderr": 0.02378858355165854 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.32, + "acc_norm_stderr": 0.046882617226215034 + }, + "hendrycksTest-astronomy": { + "acc": 0.19078947368421054, + "acc_stderr": 0.031975658210325, + "acc_norm": 0.35526315789473684, + "acc_norm_stderr": 0.038947344870133176 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.470810777559055, + "likelihood_difference_stderr": 0.18512745233861563, + "pct_stereotype": 0.4547244094488189, + "pct_stereotype_stderr": 0.022114553870695303 + }, + "hendrycksTest-professional_law": { + "acc": 0.2529335071707953, + "acc_stderr": 0.011102268713839989, + "acc_norm": 0.29139504563233376, + "acc_norm_stderr": 0.011605720214257598 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.24074074074074073, + "acc_stderr": 0.029157522184605607, + "acc_norm": 0.2824074074074074, + "acc_norm_stderr": 0.030701372111510927 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.25165562913907286, + "acc_stderr": 0.03543304234389985, + "acc_norm": 0.2052980132450331, + "acc_norm_stderr": 0.032979866484738364 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.24, + "acc_stderr": 0.04292346959909283, + "acc_norm": 0.36, + "acc_norm_stderr": 0.04824181513244218 + }, + "arc_challenge": { + "acc": 0.18003412969283278, + "acc_stderr": 0.011227856729050046, + "acc_norm": 0.22098976109215018, + "acc_norm_stderr": 0.012124929206818258 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.1962962962962963, + "acc_stderr": 0.024217421327417145, + "acc_norm": 0.25555555555555554, + "acc_norm_stderr": 0.026593939101844072 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.662746710526316, + "likelihood_difference_stderr": 0.2626516995166995, + "pct_stereotype": 0.6421052631578947, + "pct_stereotype_stderr": 0.03486983309720002 + }, + "hendrycksTest-global_facts": { + "acc": 0.2, + "acc_stderr": 0.04020151261036846, + "acc_norm": 0.2, + "acc_norm_stderr": 0.04020151261036846 + }, + "hendrycksTest-computer_security": { + "acc": 0.2, + "acc_stderr": 0.040201512610368445, + "acc_norm": 0.27, + "acc_norm_stderr": 0.044619604333847394 + }, + "hendrycksTest-world_religions": { + "acc": 0.21637426900584794, + "acc_stderr": 0.03158149539338733, + "acc_norm": 0.30409356725146197, + "acc_norm_stderr": 0.03528211258245231 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 8.092144268774703, + "likelihood_difference_stderr": 0.42633339700730977, + "pct_stereotype": 0.2766798418972332, + "pct_stereotype_stderr": 0.028180829560220628 + }, + "hendrycksTest-college_physics": { + "acc": 0.19607843137254902, + "acc_stderr": 0.03950581861179964, + "acc_norm": 0.23529411764705882, + "acc_norm_stderr": 0.04220773659171453 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.744791666666667, + "likelihood_difference_stderr": 0.47731032137153334, + "pct_stereotype": 0.4666666666666667, + "pct_stereotype_stderr": 0.05288198530254015 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.34, + "acc_stderr": 0.04760952285695235, + "acc_norm": 0.32, + "acc_norm_stderr": 0.046882617226215034 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.2222222222222222, + "acc_stderr": 0.0401910747255735, + "acc_norm": 0.3333333333333333, + "acc_norm_stderr": 0.04557239513497751 + }, + "hendrycksTest-management": { + "acc": 0.1553398058252427, + "acc_stderr": 0.03586594738573975, + "acc_norm": 0.2524271844660194, + "acc_norm_stderr": 0.04301250399690877 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 5.010682397959184, + "likelihood_difference_stderr": 0.3724707056064621, + "pct_stereotype": 0.5561224489795918, + "pct_stereotype_stderr": 0.0355794719495366 + }, + "hendrycksTest-college_biology": { + "acc": 0.2847222222222222, + "acc_stderr": 0.03773809990686936, + "acc_norm": 0.2847222222222222, + "acc_norm_stderr": 0.037738099906869355 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 4.8701923076923075, + "likelihood_difference_stderr": 1.521611829942417, + "pct_stereotype": 0.46153846153846156, + "pct_stereotype_stderr": 0.14390989949130545 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23687150837988827, + "acc_stderr": 0.01421957078810399, + "acc_norm": 0.23910614525139665, + "acc_norm_stderr": 0.014265554192331158 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.25886524822695034, + "acc_stderr": 0.026129572527180848, + "acc_norm": 0.25177304964539005, + "acc_norm_stderr": 0.025892151156709405 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.6377314814814814, + "likelihood_difference_stderr": 0.25779290930529997, + "pct_stereotype": 0.4166666666666667, + "pct_stereotype_stderr": 0.03362277436608044 + }, + "hendrycksTest-human_aging": { + "acc": 0.30493273542600896, + "acc_stderr": 0.030898610882477518, + "acc_norm": 0.23318385650224216, + "acc_norm_stderr": 0.028380391147094716 + }, + "hendrycksTest-nutrition": { + "acc": 0.27124183006535946, + "acc_stderr": 0.025457756696667864, + "acc_norm": 0.3464052287581699, + "acc_norm_stderr": 0.027245613047215345 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.23030303030303031, + "acc_stderr": 0.032876667586034886, + "acc_norm": 0.28484848484848485, + "acc_norm_stderr": 0.035243908445117836 + }, + "hendrycksTest-international_law": { + "acc": 0.15702479338842976, + "acc_stderr": 0.03321244842547128, + "acc_norm": 0.35537190082644626, + "acc_norm_stderr": 0.04369236326573981 + }, + "crows_pairs_french": { + "likelihood_difference": 5.408951997614788, + "likelihood_difference_stderr": 0.13396186257220605, + "pct_stereotype": 0.43410852713178294, + "pct_stereotype_stderr": 0.012106782103996013 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.23544973544973544, + "acc_stderr": 0.02185150982203173, + "acc_norm": 0.25396825396825395, + "acc_norm_stderr": 0.022418042891113942 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.22279792746113988, + "acc_stderr": 0.030031147977641545, + "acc_norm": 0.27979274611398963, + "acc_norm_stderr": 0.03239637046735703 + }, + "arc_easy": { + "acc": 0.4090909090909091, + "acc_stderr": 0.010088775152615777, + "acc_norm": 0.3691077441077441, + "acc_norm_stderr": 0.009901987410242749 + }, + "hendrycksTest-anatomy": { + "acc": 0.24444444444444444, + "acc_stderr": 0.037125378336148665, + "acc_norm": 0.2518518518518518, + "acc_norm_stderr": 0.03749850709174022 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.20689655172413793, + "acc_stderr": 0.02850137816789395, + "acc_norm": 0.2660098522167488, + "acc_norm_stderr": 0.03108982600293752 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.25163398692810457, + "acc_stderr": 0.017555818091322267, + "acc_norm": 0.2696078431372549, + "acc_norm_stderr": 0.017952449196987862 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.8701923076923075, + "likelihood_difference_stderr": 0.31405469232033567, + "pct_stereotype": 0.4945054945054945, + "pct_stereotype_stderr": 0.05270144531112881 + }, + "hendrycksTest-virology": { + "acc": 0.22289156626506024, + "acc_stderr": 0.03240004825594686, + "acc_norm": 0.23493975903614459, + "acc_norm_stderr": 0.03300533186128922 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.2656449553001277, + "acc_stderr": 0.015794302487888708, + "acc_norm": 0.23627075351213284, + "acc_norm_stderr": 0.015190473717037514 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.29770992366412213, + "acc_stderr": 0.040103589424622034, + "acc_norm": 0.2748091603053435, + "acc_norm_stderr": 0.039153454088478354 + }, + "hendrycksTest-college_medicine": { + "acc": 0.20809248554913296, + "acc_stderr": 0.03095289021774988, + "acc_norm": 0.2947976878612717, + "acc_norm_stderr": 0.03476599607516478 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 4.415217391304348, + "likelihood_difference_stderr": 0.5630298225735354, + "pct_stereotype": 0.591304347826087, + "pct_stereotype_stderr": 0.04604188749503789 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.24539877300613497, + "acc_stderr": 0.03380939813943354, + "acc_norm": 0.25766871165644173, + "acc_norm_stderr": 0.03436150827846917 + }, + "hendrycksTest-security_studies": { + "acc": 0.3142857142857143, + "acc_stderr": 0.029719329422417465, + "acc_norm": 0.27346938775510204, + "acc_norm_stderr": 0.02853556033712845 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 4.951923076923077, + "likelihood_difference_stderr": 0.5476034283871334, + "pct_stereotype": 0.6307692307692307, + "pct_stereotype_stderr": 0.06032456592830046 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.24, + "acc_stderr": 0.04292346959909283, + "acc_norm": 0.34, + "acc_norm_stderr": 0.047609522856952365 + }, + "sciq": { + "acc": 0.696, + "acc_stderr": 0.014553205687950425, + "acc_norm": 0.63, + "acc_norm_stderr": 0.015275252316519362 + }, + "hendrycksTest-business_ethics": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.23897058823529413, + "acc_stderr": 0.02590528064489301, + "acc_norm": 0.25, + "acc_norm_stderr": 0.026303648393696036 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.2967741935483871, + "acc_stderr": 0.025988500792411884, + "acc_norm": 0.3419354838709677, + "acc_norm_stderr": 0.026985289576552725 + }, + "lambada_openai": { + "ppl": 117.09850923121336, + "ppl_stderr": 4.944891089328104, + "acc": 0.22627595575392975, + "acc_stderr": 0.0058294062654043795 + }, + "hendrycksTest-marketing": { + "acc": 0.2692307692307692, + "acc_stderr": 0.029058588303748845, + "acc_norm": 0.2905982905982906, + "acc_norm_stderr": 0.029745048572674047 + }, + "hendrycksTest-machine_learning": { + "acc": 0.25, + "acc_stderr": 0.04109974682633932, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04109974682633932 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.844618055555555, + "likelihood_difference_stderr": 0.6600500505258997, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.05933908290969268 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.23737373737373738, + "acc_stderr": 0.03031371053819889, + "acc_norm": 0.2878787878787879, + "acc_norm_stderr": 0.03225883512300993 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.879261363636363, + "likelihood_difference_stderr": 0.6836856088759977, + "pct_stereotype": 0.48484848484848486, + "pct_stereotype_stderr": 0.06198888629778894 + }, + "logiqa": { + "acc": 0.21351766513056836, + "acc_stderr": 0.016073287529685214, + "acc_norm": 0.25960061443932414, + "acc_norm_stderr": 0.01719607000818003 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.21509433962264152, + "acc_stderr": 0.025288394502891363, + "acc_norm": 0.3283018867924528, + "acc_norm_stderr": 0.02890159361241178 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 5.4838598901098905, + "likelihood_difference_stderr": 0.4889141739230861, + "pct_stereotype": 0.7472527472527473, + "pct_stereotype_stderr": 0.04580951853732889 + }, + "hendrycksTest-sociology": { + "acc": 0.26865671641791045, + "acc_stderr": 0.031343283582089536, + "acc_norm": 0.27860696517412936, + "acc_norm_stderr": 0.031700561834973086 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.544976635514018, + "likelihood_difference_stderr": 0.22578758422298179, + "pct_stereotype": 0.5046728971962616, + "pct_stereotype_stderr": 0.02794962902436013 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.2, + "acc_stderr": 0.04020151261036845, + "acc_norm": 0.23, + "acc_norm_stderr": 0.04229525846816507 + }, + "hendrycksTest-econometrics": { + "acc": 0.2894736842105263, + "acc_stderr": 0.04266339443159394, + "acc_norm": 0.2719298245614035, + "acc_norm_stderr": 0.041857744240220554 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.28085106382978725, + "acc_stderr": 0.02937917046412482, + "acc_norm": 0.20425531914893616, + "acc_norm_stderr": 0.02635515841334942 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.25, + "acc_stderr": 0.03039153369274154, + "acc_norm": 0.27450980392156865, + "acc_norm_stderr": 0.031321798030832904 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.451295045045045, + "likelihood_difference_stderr": 0.4081953291405577, + "pct_stereotype": 0.6306306306306306, + "pct_stereotype_stderr": 0.04601735229444767 + } + }, + "versions": { + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-high_school_microeconomics": 0, + "crows_pairs_english_gender": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-college_computer_science": 0, + "crows_pairs_french_race_color": 0, + "winogrande": 0, + "hendrycksTest-philosophy": 0, + "crows_pairs_english_physical_appearance": 0, + "piqa": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-moral_disputes": 0, + "crows_pairs_english": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-astronomy": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-us_foreign_policy": 0, + "arc_challenge": 0, + "hendrycksTest-high_school_mathematics": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-college_physics": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-management": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-college_biology": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-professional_accounting": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-international_law": 0, + "crows_pairs_french": 0, + "hendrycksTest-elementary_mathematics": 0, + "wsc": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "arc_easy": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-professional_psychology": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-security_studies": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-abstract_algebra": 0, + "sciq": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-high_school_biology": 0, + "lambada_openai": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-machine_learning": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-high_school_geography": 0, + "crows_pairs_french_disability": 0, + "logiqa": 0, + "hendrycksTest-clinical_knowledge": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-sociology": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_english_religion": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step73000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:7", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step8.json b/data/checkpoint_eval/eleutherai_evals/step8.json new file mode 100644 index 0000000000000000000000000000000000000000..a6513090af8b819282b177367cab3e2fe2748cd4 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step8.json @@ -0,0 +1,622 @@ +{ + "results": { + "arc_easy": { + "acc": 0.26641414141414144, + "acc_stderr": 0.009071357971078687, + "acc_norm": 0.2537878787878788, + "acc_norm_stderr": 0.008929657065808295 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 14.844696969696969, + "likelihood_difference_stderr": 1.4724037604097784, + "pct_stereotype": 0.36363636363636365, + "pct_stereotype_stderr": 0.05966637484671757 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.19327731092436976, + "acc_stderr": 0.025649470265889193, + "acc_norm": 0.2815126050420168, + "acc_norm_stderr": 0.029213549414372163 + }, + "wsc": { + "acc": 0.6153846153846154, + "acc_stderr": 0.0479366886807504 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.1986754966887417, + "acc_stderr": 0.03257847384436777, + "acc_norm": 0.31788079470198677, + "acc_norm_stderr": 0.038020397601079024 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.1574074074074074, + "acc_stderr": 0.03520703990517965, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04186091791394607 + }, + "hendrycksTest-world_religions": { + "acc": 0.2046783625730994, + "acc_stderr": 0.030944459778533193, + "acc_norm": 0.24561403508771928, + "acc_norm_stderr": 0.03301405946987249 + }, + "crows_pairs_french": { + "likelihood_difference": 10.034911672629695, + "likelihood_difference_stderr": 0.23396856909749553, + "pct_stereotype": 0.5766249254621347, + "pct_stereotype_stderr": 0.012069029300507982 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 9.03695652173913, + "likelihood_difference_stderr": 0.37564611061761194, + "pct_stereotype": 0.6543478260869565, + "pct_stereotype_stderr": 0.02219819363895969 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.22592592592592592, + "acc_stderr": 0.02549753263960955, + "acc_norm": 0.26296296296296295, + "acc_norm_stderr": 0.02684205787383371 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 13.044244260204081, + "likelihood_difference_stderr": 0.8616384783016274, + "pct_stereotype": 0.413265306122449, + "pct_stereotype_stderr": 0.035262902194360866 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 5.86936936936937, + "likelihood_difference_stderr": 0.712342266308724, + "pct_stereotype": 0.5225225225225225, + "pct_stereotype_stderr": 0.04762473917649626 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036624 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2730496453900709, + "acc_stderr": 0.026577860943307857, + "acc_norm": 0.2553191489361702, + "acc_norm_stderr": 0.026011992930902006 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2, + "acc_stderr": 0.0333333333333333, + "acc_norm": 0.18620689655172415, + "acc_norm_stderr": 0.03243946159004616 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 6.25390625, + "likelihood_difference_stderr": 0.8484057131433209, + "pct_stereotype": 0.5277777777777778, + "pct_stereotype_stderr": 0.05924743948371487 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 7.038461538461538, + "likelihood_difference_stderr": 1.9443187814541079, + "pct_stereotype": 0.38461538461538464, + "pct_stereotype_stderr": 0.1404416814115811 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.17721518987341772, + "acc_stderr": 0.024856364184503217, + "acc_norm": 0.2109704641350211, + "acc_norm_stderr": 0.02655837250266192 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.17, + "acc_stderr": 0.0377525168068637, + "acc_norm": 0.19, + "acc_norm_stderr": 0.03942772444036623 + }, + "lambada_openai": { + "ppl": 3646061.8456566, + "ppl_stderr": 355236.72805614787, + "acc": 0.0, + "acc_stderr": 0.0 + }, + "crows_pairs_french_age": { + "likelihood_difference": 5.894444444444445, + "likelihood_difference_stderr": 0.7650007372090334, + "pct_stereotype": 0.6555555555555556, + "pct_stereotype_stderr": 0.050369697187736755 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 5.904281496062992, + "likelihood_difference_stderr": 0.36247868571395325, + "pct_stereotype": 0.38188976377952755, + "pct_stereotype_stderr": 0.021577344577442634 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.17575757575757575, + "acc_stderr": 0.02972094300622445, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.030117688929503585 + }, + "piqa": { + "acc": 0.5239390642002176, + "acc_stderr": 0.011652445621079262, + "acc_norm": 0.5195865070729053, + "acc_norm_stderr": 0.011656869979288458 + }, + "hendrycksTest-anatomy": { + "acc": 0.22962962962962963, + "acc_stderr": 0.03633384414073465, + "acc_norm": 0.2962962962962963, + "acc_norm_stderr": 0.03944624162501117 + }, + "crows_pairs_english": { + "likelihood_difference": 5.7787343470483, + "likelihood_difference_stderr": 0.1995147197033885, + "pct_stereotype": 0.4639236732259988, + "pct_stereotype_stderr": 0.012181466483312616 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.19622641509433963, + "acc_stderr": 0.02444238813110085, + "acc_norm": 0.2943396226415094, + "acc_norm_stderr": 0.02804918631569524 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.23232323232323232, + "acc_stderr": 0.03008862949021749, + "acc_norm": 0.2777777777777778, + "acc_norm_stderr": 0.03191178226713547 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 6.42115162037037, + "likelihood_difference_stderr": 0.536959840207466, + "pct_stereotype": 0.44907407407407407, + "pct_stereotype_stderr": 0.03392238405321617 + }, + "hendrycksTest-security_studies": { + "acc": 0.3183673469387755, + "acc_stderr": 0.02982253379398205, + "acc_norm": 0.19591836734693877, + "acc_norm_stderr": 0.025409301953225678 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.2426470588235294, + "acc_stderr": 0.026040662474201264, + "acc_norm": 0.2757352941176471, + "acc_norm_stderr": 0.027146271936625162 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2595419847328244, + "acc_stderr": 0.03844876139785271, + "acc_norm": 0.2824427480916031, + "acc_norm_stderr": 0.03948406125768362 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2680851063829787, + "acc_stderr": 0.028957342788342347, + "acc_norm": 0.251063829787234, + "acc_norm_stderr": 0.02834696377716245 + }, + "hendrycksTest-philosophy": { + "acc": 0.22508038585209003, + "acc_stderr": 0.023720088516179027, + "acc_norm": 0.29260450160771706, + "acc_norm_stderr": 0.02583989833487798 + }, + "hendrycksTest-public_relations": { + "acc": 0.2545454545454545, + "acc_stderr": 0.041723430387053825, + "acc_norm": 0.18181818181818182, + "acc_norm_stderr": 0.03694284335337798 + }, + "hendrycksTest-sociology": { + "acc": 0.26865671641791045, + "acc_stderr": 0.031343283582089536, + "acc_norm": 0.2736318407960199, + "acc_norm_stderr": 0.03152439186555402 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 7.9192307692307695, + "likelihood_difference_stderr": 1.2408777134628783, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.06231481440776789 + }, + "hendrycksTest-human_aging": { + "acc": 0.28699551569506726, + "acc_stderr": 0.030360379710291936, + "acc_norm": 0.26905829596412556, + "acc_norm_stderr": 0.029763779406874975 + }, + "hendrycksTest-virology": { + "acc": 0.19879518072289157, + "acc_stderr": 0.03106939026078942, + "acc_norm": 0.21686746987951808, + "acc_norm_stderr": 0.03208284450356365 + }, + "hendrycksTest-college_physics": { + "acc": 0.23529411764705882, + "acc_stderr": 0.04220773659171451, + "acc_norm": 0.22549019607843138, + "acc_norm_stderr": 0.04158307533083286 + }, + "hendrycksTest-prehistory": { + "acc": 0.24074074074074073, + "acc_stderr": 0.023788583551658544, + "acc_norm": 0.21604938271604937, + "acc_norm_stderr": 0.022899162918445792 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.23, + "acc_stderr": 0.04229525846816507, + "acc_norm": 0.22, + "acc_norm_stderr": 0.041633319989322695 + }, + "hendrycksTest-astronomy": { + "acc": 0.17105263157894737, + "acc_stderr": 0.030643607071677098, + "acc_norm": 0.23026315789473684, + "acc_norm_stderr": 0.03426059424403165 + }, + "winogrande": { + "acc": 0.489344909234412, + "acc_stderr": 0.0140492945362904 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.24074074074074073, + "acc_stderr": 0.029157522184605593, + "acc_norm": 0.28703703703703703, + "acc_norm_stderr": 0.030851992993257013 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.2254335260115607, + "acc_stderr": 0.02249723019096755, + "acc_norm": 0.2138728323699422, + "acc_norm_stderr": 0.022075709251757173 + }, + "hendrycksTest-econometrics": { + "acc": 0.24561403508771928, + "acc_stderr": 0.04049339297748139, + "acc_norm": 0.3157894736842105, + "acc_norm_stderr": 0.04372748290278007 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 7.683216510903427, + "likelihood_difference_stderr": 0.45391062504811047, + "pct_stereotype": 0.5327102803738317, + "pct_stereotype_stderr": 0.027890972865217984 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.2128205128205128, + "acc_stderr": 0.020752423722128016, + "acc_norm": 0.24358974358974358, + "acc_norm_stderr": 0.02176373368417392 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.18627450980392157, + "acc_stderr": 0.027325470966716326, + "acc_norm": 0.2647058823529412, + "acc_norm_stderr": 0.030964517926923393 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 6.198863636363637, + "likelihood_difference_stderr": 3.164913494502804, + "pct_stereotype": 0.45454545454545453, + "pct_stereotype_stderr": 0.15745916432444335 + }, + "hendrycksTest-nutrition": { + "acc": 0.17973856209150327, + "acc_stderr": 0.021986032182064148, + "acc_norm": 0.24509803921568626, + "acc_norm_stderr": 0.024630048979824782 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.1656441717791411, + "acc_stderr": 0.029208296231259104, + "acc_norm": 0.2331288343558282, + "acc_norm_stderr": 0.03322015795776741 + }, + "hendrycksTest-marketing": { + "acc": 0.23076923076923078, + "acc_stderr": 0.027601921381417597, + "acc_norm": 0.2564102564102564, + "acc_norm_stderr": 0.028605953702004243 + }, + "logiqa": { + "acc": 0.23195084485407066, + "acc_stderr": 0.016555252497925898, + "acc_norm": 0.2488479262672811, + "acc_norm_stderr": 0.016957985904525585 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.23015873015873015, + "acc_stderr": 0.02167921966369314, + "acc_norm": 0.23809523809523808, + "acc_norm_stderr": 0.021935878081184763 + }, + "hendrycksTest-international_law": { + "acc": 0.09090909090909091, + "acc_stderr": 0.02624319405407387, + "acc_norm": 0.2231404958677686, + "acc_norm_stderr": 0.03800754475228733 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 14.885302197802197, + "likelihood_difference_stderr": 1.0262045022589799, + "pct_stereotype": 0.7912087912087912, + "pct_stereotype_stderr": 0.04284305206509431 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.26436781609195403, + "acc_stderr": 0.015769984840690518, + "acc_norm": 0.2656449553001277, + "acc_norm_stderr": 0.01579430248788872 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-college_biology": { + "acc": 0.2638888888888889, + "acc_stderr": 0.03685651095897532, + "acc_norm": 0.2708333333333333, + "acc_norm_stderr": 0.03716177437566017 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.21, + "acc_stderr": 0.040936018074033256, + "acc_norm": 0.17, + "acc_norm_stderr": 0.03775251680686371 + }, + "crows_pairs_english_age": { + "likelihood_difference": 3.793956043956044, + "likelihood_difference_stderr": 0.6669967521493633, + "pct_stereotype": 0.6373626373626373, + "pct_stereotype_stderr": 0.05067669921031868 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.22279792746113988, + "acc_stderr": 0.03003114797764154, + "acc_norm": 0.24870466321243523, + "acc_norm_stderr": 0.03119584087770029 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 10.175347222222221, + "likelihood_difference_stderr": 1.3645954505299696, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.05933908290969268 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.23798882681564246, + "acc_stderr": 0.014242630070574915, + "acc_norm": 0.27262569832402234, + "acc_norm_stderr": 0.014893391735249588 + }, + "hendrycksTest-management": { + "acc": 0.23300970873786409, + "acc_stderr": 0.041858325989283136, + "acc_norm": 0.2815533980582524, + "acc_norm_stderr": 0.04453254836326466 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.21568627450980393, + "acc_stderr": 0.016639319350313264, + "acc_norm": 0.22875816993464052, + "acc_norm_stderr": 0.016992723465466243 + }, + "arc_challenge": { + "acc": 0.21075085324232082, + "acc_stderr": 0.011918271754852171, + "acc_norm": 0.24488054607508533, + "acc_norm_stderr": 0.012566273985131354 + }, + "hendrycksTest-professional_law": { + "acc": 0.2333767926988266, + "acc_stderr": 0.010803108481179099, + "acc_norm": 0.24315514993481094, + "acc_norm_stderr": 0.010956556654417355 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 6.031578947368421, + "likelihood_difference_stderr": 0.5493093736965833, + "pct_stereotype": 0.49473684210526314, + "pct_stereotype_stderr": 0.03636763337787883 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.21674876847290642, + "acc_stderr": 0.028990331252516235, + "acc_norm": 0.21674876847290642, + "acc_norm_stderr": 0.028990331252516235 + }, + "hendrycksTest-formal_logic": { + "acc": 0.2619047619047619, + "acc_stderr": 0.03932537680392872, + "acc_norm": 0.2619047619047619, + "acc_norm_stderr": 0.039325376803928724 + }, + "hendrycksTest-machine_learning": { + "acc": 0.33035714285714285, + "acc_stderr": 0.04464285714285714, + "acc_norm": 0.32142857142857145, + "acc_norm_stderr": 0.044328040552915185 + }, + "hendrycksTest-college_medicine": { + "acc": 0.2138728323699422, + "acc_stderr": 0.03126511206173044, + "acc_norm": 0.26011560693641617, + "acc_norm_stderr": 0.033450369167889925 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 5.76377688172043, + "likelihood_difference_stderr": 0.7836824367448326, + "pct_stereotype": 0.45161290322580644, + "pct_stereotype_stderr": 0.051883930752016603 + }, + "hendrycksTest-computer_security": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.32, + "acc_norm_stderr": 0.046882617226215034 + }, + "sciq": { + "acc": 0.201, + "acc_stderr": 0.012679107214617322, + "acc_norm": 0.212, + "acc_norm_stderr": 0.012931481864938052 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.21935483870967742, + "acc_stderr": 0.02354079935872329, + "acc_norm": 0.22258064516129034, + "acc_norm_stderr": 0.02366421667164251 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 10.010128458498023, + "likelihood_difference_stderr": 0.5534217267350656, + "pct_stereotype": 0.5770750988142292, + "pct_stereotype_stderr": 0.031120568731718617 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.36, + "acc_norm_stderr": 0.04824181513244218 + }, + "hendrycksTest-global_facts": { + "acc": 0.28, + "acc_stderr": 0.045126085985421276, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542126 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.22, + "acc_norm_stderr": 0.041633319989322695 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 4.97685546875, + "likelihood_difference_stderr": 0.5099312963992724, + "pct_stereotype": 0.490625, + "pct_stereotype_stderr": 0.027989704184941015 + }, + "hendrycksTest-business_ethics": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.25137614678899084, + "acc_stderr": 0.01859920636028741, + "acc_norm": 0.24770642201834864, + "acc_norm_stderr": 0.018508143602547805 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 12.409239130434782, + "likelihood_difference_stderr": 0.9682588361060674, + "pct_stereotype": 0.6260869565217392, + "pct_stereotype_stderr": 0.045315858286449635 + } + }, + "versions": { + "arc_easy": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-high_school_microeconomics": 0, + "wsc": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-jurisprudence": 0, + "hendrycksTest-world_religions": 0, + "crows_pairs_french": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-high_school_mathematics": 0, + "crows_pairs_french_socioeconomic": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-electrical_engineering": 0, + "crows_pairs_english_physical_appearance": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-college_chemistry": 0, + "lambada_openai": 0, + "crows_pairs_french_age": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-high_school_european_history": 0, + "piqa": 0, + "hendrycksTest-anatomy": 0, + "crows_pairs_english": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-high_school_geography": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-sociology": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-virology": 0, + "hendrycksTest-college_physics": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-astronomy": 0, + "winogrande": 0, + "hendrycksTest-high_school_statistics": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-econometrics": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-high_school_us_history": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-marketing": 0, + "logiqa": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-international_law": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-us_foreign_policy": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-college_mathematics": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-moral_scenarios": 0, + "hendrycksTest-management": 0, + "hendrycksTest-professional_psychology": 0, + "arc_challenge": 0, + "hendrycksTest-professional_law": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_english_sexual_orientation": 0, + "hendrycksTest-computer_security": 0, + "sciq": 0, + "hendrycksTest-high_school_biology": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-medical_genetics": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-high_school_psychology": 0, + "crows_pairs_french_religion": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "use_accelerate=True,pretrained=EleutherAI/pythia-v1.1-70m,revision=step8", + "num_fewshot": 0, + "batch_size": 32, + "device": null, + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step83000.json b/data/checkpoint_eval/eleutherai_evals/step83000.json new file mode 100644 index 0000000000000000000000000000000000000000..05a01c581093fc5db3a4d0cbad424c5cb849d2e4 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step83000.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-miscellaneous": { + "acc": 0.26181353767560667, + "acc_stderr": 0.01572083867844526, + "acc_norm": 0.24776500638569604, + "acc_norm_stderr": 0.015438083080568965 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2978723404255319, + "acc_stderr": 0.027281608344469414, + "acc_norm": 0.2695035460992908, + "acc_norm_stderr": 0.02646903681859062 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.2346368715083799, + "acc_stderr": 0.014173044098303654, + "acc_norm": 0.2569832402234637, + "acc_norm_stderr": 0.014614465821966361 + }, + "sciq": { + "acc": 0.633, + "acc_stderr": 0.015249378464171749, + "acc_norm": 0.552, + "acc_norm_stderr": 0.01573351656634783 + }, + "hendrycksTest-nutrition": { + "acc": 0.27450980392156865, + "acc_stderr": 0.02555316999182651, + "acc_norm": 0.3366013071895425, + "acc_norm_stderr": 0.027057974624494382 + }, + "piqa": { + "acc": 0.5973884657236126, + "acc_stderr": 0.011442395233488698, + "acc_norm": 0.5854189336235038, + "acc_norm_stderr": 0.011494326682255158 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.2549019607843137, + "acc_stderr": 0.030587591351604243, + "acc_norm": 0.27941176470588236, + "acc_norm_stderr": 0.031493281045079556 + }, + "hendrycksTest-international_law": { + "acc": 0.15702479338842976, + "acc_stderr": 0.0332124484254713, + "acc_norm": 0.4132231404958678, + "acc_norm_stderr": 0.04495087843548408 + }, + "hendrycksTest-anatomy": { + "acc": 0.2222222222222222, + "acc_stderr": 0.035914440841969694, + "acc_norm": 0.2740740740740741, + "acc_norm_stderr": 0.03853254836552003 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.660533489096573, + "likelihood_difference_stderr": 0.22532366484380598, + "pct_stereotype": 0.5077881619937694, + "pct_stereotype_stderr": 0.027947458769356347 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.2757352941176471, + "acc_stderr": 0.027146271936625162, + "acc_norm": 0.3125, + "acc_norm_stderr": 0.02815637344037142 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.27339449541284405, + "acc_stderr": 0.0191092998460983, + "acc_norm": 0.24403669724770644, + "acc_norm_stderr": 0.018415286351416395 + }, + "hendrycksTest-astronomy": { + "acc": 0.23684210526315788, + "acc_stderr": 0.03459777606810535, + "acc_norm": 0.3355263157894737, + "acc_norm_stderr": 0.03842498559395268 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.22699386503067484, + "acc_stderr": 0.03291099578615769, + "acc_norm": 0.26380368098159507, + "acc_norm_stderr": 0.03462419931615623 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.6946022727272725, + "likelihood_difference_stderr": 0.7491237826255029, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.06201736729460421 + }, + "hendrycksTest-high_school_chemistry": { + "acc": 0.2315270935960591, + "acc_stderr": 0.029678333141444455, + "acc_norm": 0.3054187192118227, + "acc_norm_stderr": 0.03240661565868408 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.25925925925925924, + "acc_stderr": 0.022569897074918424, + "acc_norm": 0.25925925925925924, + "acc_norm_stderr": 0.022569897074918424 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.3053435114503817, + "acc_stderr": 0.040393149787245605, + "acc_norm": 0.22137404580152673, + "acc_norm_stderr": 0.03641297081313729 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.25, + "acc_stderr": 0.01751781884501444, + "acc_norm": 0.2630718954248366, + "acc_norm_stderr": 0.017812676542320653 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 4.175815217391304, + "likelihood_difference_stderr": 0.5425080644657401, + "pct_stereotype": 0.5130434782608696, + "pct_stereotype_stderr": 0.04681335351503156 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.28, + "acc_stderr": 0.045126085985421255, + "acc_norm": 0.29, + "acc_norm_stderr": 0.04560480215720684 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.3977272727272725, + "likelihood_difference_stderr": 1.790491828842816, + "pct_stereotype": 0.6363636363636364, + "pct_stereotype_stderr": 0.15212000482437738 + }, + "hendrycksTest-econometrics": { + "acc": 0.30701754385964913, + "acc_stderr": 0.04339138322579861, + "acc_norm": 0.2631578947368421, + "acc_norm_stderr": 0.041424397194893624 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.23529411764705882, + "acc_stderr": 0.027553614467863825, + "acc_norm": 0.3403361344537815, + "acc_norm_stderr": 0.030778057422931673 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.2774566473988439, + "acc_stderr": 0.024105712607754307, + "acc_norm": 0.2947976878612717, + "acc_norm_stderr": 0.024547617794803835 + }, + "hendrycksTest-machine_learning": { + "acc": 0.33035714285714285, + "acc_stderr": 0.04464285714285713, + "acc_norm": 0.20535714285714285, + "acc_norm_stderr": 0.03834241021419073 + }, + "hendrycksTest-management": { + "acc": 0.2621359223300971, + "acc_stderr": 0.043546310772605935, + "acc_norm": 0.27184466019417475, + "acc_norm_stderr": 0.044052680241409216 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 5.223692602040816, + "likelihood_difference_stderr": 0.3716517632652829, + "pct_stereotype": 0.5510204081632653, + "pct_stereotype_stderr": 0.03561884533975955 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.2564102564102564, + "acc_stderr": 0.022139081103971534, + "acc_norm": 0.28205128205128205, + "acc_norm_stderr": 0.022815813098896597 + }, + "hendrycksTest-security_studies": { + "acc": 0.2897959183673469, + "acc_stderr": 0.029043088683304345, + "acc_norm": 0.2530612244897959, + "acc_norm_stderr": 0.027833023871399683 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.25, + "acc_stderr": 0.04351941398892446, + "acc_norm": 0.31, + "acc_norm_stderr": 0.04648231987117316 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.2361111111111111, + "acc_stderr": 0.028963702570791033, + "acc_norm": 0.27314814814814814, + "acc_norm_stderr": 0.03038805130167812 + }, + "crows_pairs_english": { + "likelihood_difference": 3.675657796660704, + "likelihood_difference_stderr": 0.10428478695252169, + "pct_stereotype": 0.5438282647584973, + "pct_stereotype_stderr": 0.012166287275376289 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.2185430463576159, + "acc_stderr": 0.03374235550425694, + "acc_norm": 0.25165562913907286, + "acc_norm_stderr": 0.03543304234389985 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.5057713963963963, + "likelihood_difference_stderr": 0.4253117969664197, + "pct_stereotype": 0.6216216216216216, + "pct_stereotype_stderr": 0.04624128233851482 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.478158602150538, + "likelihood_difference_stderr": 0.5463367427565824, + "pct_stereotype": 0.7849462365591398, + "pct_stereotype_stderr": 0.04283507835554755 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 4.158223684210526, + "likelihood_difference_stderr": 0.2827099752616182, + "pct_stereotype": 0.5842105263157895, + "pct_stereotype_stderr": 0.0358501132552001 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 2.90234375, + "likelihood_difference_stderr": 0.26743360486517015, + "pct_stereotype": 0.5375, + "pct_stereotype_stderr": 0.02791577963000663 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.2413793103448276, + "acc_stderr": 0.03565998174135303, + "acc_norm": 0.27586206896551724, + "acc_norm_stderr": 0.03724563619774632 + }, + "hendrycksTest-business_ethics": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "hendrycksTest-global_facts": { + "acc": 0.19, + "acc_stderr": 0.03942772444036625, + "acc_norm": 0.24, + "acc_norm_stderr": 0.042923469599092816 + }, + "hendrycksTest-public_relations": { + "acc": 0.2545454545454545, + "acc_stderr": 0.04172343038705383, + "acc_norm": 0.20909090909090908, + "acc_norm_stderr": 0.03895091015724137 + }, + "crows_pairs_french_age": { + "likelihood_difference": 4.967708333333333, + "likelihood_difference_stderr": 0.4550873657608913, + "pct_stereotype": 0.43333333333333335, + "pct_stereotype_stderr": 0.05252667118728807 + }, + "hendrycksTest-virology": { + "acc": 0.25301204819277107, + "acc_stderr": 0.033844291552331346, + "acc_norm": 0.21686746987951808, + "acc_norm_stderr": 0.03208284450356365 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.224392361111111, + "likelihood_difference_stderr": 0.5949955425776441, + "pct_stereotype": 0.4861111111111111, + "pct_stereotype_stderr": 0.059316185327165566 + }, + "hendrycksTest-human_aging": { + "acc": 0.26905829596412556, + "acc_stderr": 0.02976377940687497, + "acc_norm": 0.21524663677130046, + "acc_norm_stderr": 0.027584066602208274 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.25906735751295334, + "acc_stderr": 0.03161877917935409, + "acc_norm": 0.3005181347150259, + "acc_norm_stderr": 0.033088185944157515 + }, + "hendrycksTest-philosophy": { + "acc": 0.24437299035369775, + "acc_stderr": 0.024406162094668893, + "acc_norm": 0.26688102893890675, + "acc_norm_stderr": 0.025122637608816646 + }, + "lambada_openai": { + "ppl": 124.26962204175287, + "ppl_stderr": 5.363117769801199, + "acc": 0.22627595575392975, + "acc_stderr": 0.005829406265404375 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 3.8758680555555554, + "likelihood_difference_stderr": 0.41377726625457284, + "pct_stereotype": 0.625, + "pct_stereotype_stderr": 0.05745481997211521 + }, + "winogrande": { + "acc": 0.5193370165745856, + "acc_stderr": 0.014041972733712972 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.6435908564814814, + "likelihood_difference_stderr": 0.26705840381438256, + "pct_stereotype": 0.4305555555555556, + "pct_stereotype_stderr": 0.03376922151252336 + }, + "hendrycksTest-college_physics": { + "acc": 0.17647058823529413, + "acc_stderr": 0.03793281185307809, + "acc_norm": 0.23529411764705882, + "acc_norm_stderr": 0.04220773659171453 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.7424950787401574, + "likelihood_difference_stderr": 0.18169346622004526, + "pct_stereotype": 0.5059055118110236, + "pct_stereotype_stderr": 0.02220423067397246 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.2936170212765957, + "acc_stderr": 0.02977164271249123, + "acc_norm": 0.1829787234042553, + "acc_norm_stderr": 0.025276041000449966 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.23773584905660378, + "acc_stderr": 0.026199808807561915, + "acc_norm": 0.3018867924528302, + "acc_norm_stderr": 0.028254200344438662 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.18, + "acc_stderr": 0.03861229196653697, + "acc_norm": 0.2, + "acc_norm_stderr": 0.04020151261036845 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "hendrycksTest-computer_security": { + "acc": 0.22, + "acc_stderr": 0.041633319989322716, + "acc_norm": 0.32, + "acc_norm_stderr": 0.04688261722621503 + }, + "hendrycksTest-world_religions": { + "acc": 0.23976608187134502, + "acc_stderr": 0.03274485211946956, + "acc_norm": 0.3157894736842105, + "acc_norm_stderr": 0.03565079670708311 + }, + "hendrycksTest-sociology": { + "acc": 0.24378109452736318, + "acc_stderr": 0.030360490154014638, + "acc_norm": 0.2835820895522388, + "acc_norm_stderr": 0.03187187537919798 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.32, + "acc_stderr": 0.04688261722621504, + "acc_norm": 0.33, + "acc_norm_stderr": 0.047258156262526045 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.2109704641350211, + "acc_stderr": 0.02655837250266192, + "acc_norm": 0.2742616033755274, + "acc_norm_stderr": 0.029041333510598046 + }, + "logiqa": { + "acc": 0.23195084485407066, + "acc_stderr": 0.0165552524979259, + "acc_norm": 0.27035330261136714, + "acc_norm_stderr": 0.01742069478339314 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.2, + "acc_stderr": 0.04020151261036843, + "acc_norm": 0.26, + "acc_norm_stderr": 0.044084400227680814 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.2870967741935484, + "acc_stderr": 0.025736542745594528, + "acc_norm": 0.3, + "acc_norm_stderr": 0.02606936229533513 + }, + "hendrycksTest-marketing": { + "acc": 0.27350427350427353, + "acc_stderr": 0.029202540153431177, + "acc_norm": 0.2606837606837607, + "acc_norm_stderr": 0.028760348956523414 + }, + "hendrycksTest-professional_law": { + "acc": 0.24771838331160365, + "acc_stderr": 0.011025499291443742, + "acc_norm": 0.27444589308996087, + "acc_norm_stderr": 0.011397043163078154 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 7.527667984189724, + "likelihood_difference_stderr": 0.4209795564667756, + "pct_stereotype": 0.308300395256917, + "pct_stereotype_stderr": 0.02909012143059231 + }, + "hendrycksTest-prehistory": { + "acc": 0.26851851851851855, + "acc_stderr": 0.024659685185967284, + "acc_norm": 0.21296296296296297, + "acc_norm_stderr": 0.0227797190887334 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 7.548076923076923, + "likelihood_difference_stderr": 0.5113727094452629, + "pct_stereotype": 0.8131868131868132, + "pct_stereotype_stderr": 0.04108446855035883 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.1814814814814815, + "acc_stderr": 0.023499264669407292, + "acc_norm": 0.22962962962962963, + "acc_norm_stderr": 0.025644108639267613 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.2037037037037037, + "acc_stderr": 0.038935425188248475, + "acc_norm": 0.3611111111111111, + "acc_norm_stderr": 0.04643454608906275 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.847758152173913, + "likelihood_difference_stderr": 0.2507391728199927, + "pct_stereotype": 0.3239130434782609, + "pct_stereotype_stderr": 0.021842842500532617 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.24, + "acc_stderr": 0.04292346959909283, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909283 + }, + "arc_easy": { + "acc": 0.37415824915824913, + "acc_stderr": 0.009929516948977625, + "acc_norm": 0.3367003367003367, + "acc_norm_stderr": 0.009697166595752477 + }, + "arc_challenge": { + "acc": 0.18600682593856654, + "acc_stderr": 0.011370940183266749, + "acc_norm": 0.22440273037542663, + "acc_norm_stderr": 0.012191404938603843 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.8133585164835164, + "likelihood_difference_stderr": 0.27309263450343635, + "pct_stereotype": 0.4725274725274725, + "pct_stereotype_stderr": 0.05262501097748859 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.492307692307692, + "likelihood_difference_stderr": 0.571991498636384, + "pct_stereotype": 0.6461538461538462, + "pct_stereotype_stderr": 0.05977027026123099 + }, + "crows_pairs_french": { + "likelihood_difference": 5.452854800238521, + "likelihood_difference_stderr": 0.13262546821335017, + "pct_stereotype": 0.4442456768038163, + "pct_stereotype_stderr": 0.012137130534698507 + }, + "hendrycksTest-formal_logic": { + "acc": 0.2777777777777778, + "acc_stderr": 0.040061680838488774, + "acc_norm": 0.29365079365079366, + "acc_norm_stderr": 0.04073524322147125 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.18181818181818182, + "acc_stderr": 0.030117688929503585, + "acc_norm": 0.2606060606060606, + "acc_norm_stderr": 0.03427743175816524 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.2878787878787879, + "acc_stderr": 0.03225883512300992, + "acc_norm": 0.3181818181818182, + "acc_norm_stderr": 0.03318477333845331 + }, + "hendrycksTest-college_medicine": { + "acc": 0.24277456647398843, + "acc_stderr": 0.0326926380614177, + "acc_norm": 0.3063583815028902, + "acc_norm_stderr": 0.03514942551267437 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 4.454326923076923, + "likelihood_difference_stderr": 1.3817380041698064, + "pct_stereotype": 0.5384615384615384, + "pct_stereotype_stderr": 0.14390989949130545 + }, + "hendrycksTest-college_biology": { + "acc": 0.2569444444444444, + "acc_stderr": 0.03653946969442099, + "acc_norm": 0.2777777777777778, + "acc_norm_stderr": 0.037455547914624576 + } + }, + "versions": { + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-moral_scenarios": 0, + "sciq": 0, + "hendrycksTest-nutrition": 0, + "piqa": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-international_law": 0, + "hendrycksTest-anatomy": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-professional_medicine": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-logical_fallacies": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-high_school_chemistry": 0, + "hendrycksTest-elementary_mathematics": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-professional_psychology": 0, + "crows_pairs_french_religion": 0, + "hendrycksTest-college_computer_science": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-moral_disputes": 0, + "hendrycksTest-machine_learning": 0, + "hendrycksTest-management": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-high_school_statistics": 0, + "crows_pairs_english": 0, + "hendrycksTest-high_school_physics": 0, + "crows_pairs_english_religion": 0, + "crows_pairs_english_sexual_orientation": 0, + "crows_pairs_english_socioeconomic": 0, + "crows_pairs_english_gender": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-business_ethics": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-public_relations": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-virology": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-philosophy": 0, + "lambada_openai": 0, + "crows_pairs_english_physical_appearance": 0, + "winogrande": 0, + "crows_pairs_english_nationality": 0, + "hendrycksTest-college_physics": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-abstract_algebra": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-high_school_world_history": 0, + "logiqa": 0, + "wsc": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-high_school_biology": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-professional_law": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-prehistory": 0, + "crows_pairs_french_sexual_orientation": 0, + "hendrycksTest-high_school_mathematics": 0, + "hendrycksTest-jurisprudence": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-us_foreign_policy": 0, + "arc_easy": 0, + "arc_challenge": 0, + "crows_pairs_english_age": 0, + "crows_pairs_english_disability": 0, + "crows_pairs_french": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-high_school_european_history": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-college_biology": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step83000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:0", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/eleutherai_evals/step93000.json b/data/checkpoint_eval/eleutherai_evals/step93000.json new file mode 100644 index 0000000000000000000000000000000000000000..f525068856816e9713105ede58f2c54aceea2f73 --- /dev/null +++ b/data/checkpoint_eval/eleutherai_evals/step93000.json @@ -0,0 +1,622 @@ +{ + "results": { + "hendrycksTest-high_school_chemistry": { + "acc": 0.19704433497536947, + "acc_stderr": 0.02798672466673622, + "acc_norm": 0.26108374384236455, + "acc_norm_stderr": 0.03090379695211447 + }, + "crows_pairs_english_disability": { + "likelihood_difference": 5.444230769230769, + "likelihood_difference_stderr": 0.5923616154197596, + "pct_stereotype": 0.6153846153846154, + "pct_stereotype_stderr": 0.06081303192631497 + }, + "hendrycksTest-marketing": { + "acc": 0.2564102564102564, + "acc_stderr": 0.028605953702004253, + "acc_norm": 0.2863247863247863, + "acc_norm_stderr": 0.02961432369045665 + }, + "hendrycksTest-moral_disputes": { + "acc": 0.24855491329479767, + "acc_stderr": 0.023267528432100174, + "acc_norm": 0.31213872832369943, + "acc_norm_stderr": 0.02494679222527231 + }, + "crows_pairs_french_physical_appearance": { + "likelihood_difference": 5.701822916666667, + "likelihood_difference_stderr": 0.6164179150786165, + "pct_stereotype": 0.5, + "pct_stereotype_stderr": 0.05933908290969268 + }, + "hendrycksTest-astronomy": { + "acc": 0.2565789473684211, + "acc_stderr": 0.035541803680256896, + "acc_norm": 0.3815789473684211, + "acc_norm_stderr": 0.03953173377749194 + }, + "hendrycksTest-professional_law": { + "acc": 0.2607561929595828, + "acc_stderr": 0.011213471559602336, + "acc_norm": 0.2777053455019557, + "acc_norm_stderr": 0.01143874142276956 + }, + "hendrycksTest-high_school_world_history": { + "acc": 0.2320675105485232, + "acc_stderr": 0.02747974455080852, + "acc_norm": 0.2616033755274262, + "acc_norm_stderr": 0.028609516716994934 + }, + "hendrycksTest-elementary_mathematics": { + "acc": 0.2566137566137566, + "acc_stderr": 0.022494510767503154, + "acc_norm": 0.2698412698412698, + "acc_norm_stderr": 0.022860838309232072 + }, + "crows_pairs_french_autre": { + "likelihood_difference": 5.016826923076923, + "likelihood_difference_stderr": 1.136143920954686, + "pct_stereotype": 0.46153846153846156, + "pct_stereotype_stderr": 0.14390989949130548 + }, + "hendrycksTest-high_school_computer_science": { + "acc": 0.22, + "acc_stderr": 0.0416333199893227, + "acc_norm": 0.31, + "acc_norm_stderr": 0.046482319871173156 + }, + "hendrycksTest-logical_fallacies": { + "acc": 0.1901840490797546, + "acc_stderr": 0.03083349114628123, + "acc_norm": 0.2822085889570552, + "acc_norm_stderr": 0.03536117886664743 + }, + "hendrycksTest-prehistory": { + "acc": 0.2962962962962963, + "acc_stderr": 0.025407197798890165, + "acc_norm": 0.22839506172839505, + "acc_norm_stderr": 0.023358211840626267 + }, + "hendrycksTest-machine_learning": { + "acc": 0.24107142857142858, + "acc_stderr": 0.040598672469526864, + "acc_norm": 0.21428571428571427, + "acc_norm_stderr": 0.03894641120044792 + }, + "wsc": { + "acc": 0.36538461538461536, + "acc_stderr": 0.0474473339327792 + }, + "hendrycksTest-econometrics": { + "acc": 0.3157894736842105, + "acc_stderr": 0.04372748290278007, + "acc_norm": 0.2719298245614035, + "acc_norm_stderr": 0.041857744240220554 + }, + "hendrycksTest-high_school_statistics": { + "acc": 0.24537037037037038, + "acc_stderr": 0.029346665094372955, + "acc_norm": 0.2638888888888889, + "acc_norm_stderr": 0.03005820270430985 + }, + "crows_pairs_french_religion": { + "likelihood_difference": 4.592934782608696, + "likelihood_difference_stderr": 0.5258696424725507, + "pct_stereotype": 0.591304347826087, + "pct_stereotype_stderr": 0.04604188749503789 + }, + "crows_pairs_french_sexual_orientation": { + "likelihood_difference": 6.3279532967032965, + "likelihood_difference_stderr": 0.4838492961401693, + "pct_stereotype": 0.7912087912087912, + "pct_stereotype_stderr": 0.04284305206509431 + }, + "crows_pairs_french_socioeconomic": { + "likelihood_difference": 5.059749681122449, + "likelihood_difference_stderr": 0.4036977090086114, + "pct_stereotype": 0.5459183673469388, + "pct_stereotype_stderr": 0.035654431417332814 + }, + "hendrycksTest-virology": { + "acc": 0.28313253012048195, + "acc_stderr": 0.03507295431370519, + "acc_norm": 0.22289156626506024, + "acc_norm_stderr": 0.03240004825594688 + }, + "arc_easy": { + "acc": 0.359006734006734, + "acc_stderr": 0.009843424713072176, + "acc_norm": 0.3514309764309764, + "acc_norm_stderr": 0.00979639558281772 + }, + "logiqa": { + "acc": 0.2196620583717358, + "acc_stderr": 0.01623910941493396, + "acc_norm": 0.2642089093701997, + "acc_norm_stderr": 0.017293954549744518 + }, + "hendrycksTest-abstract_algebra": { + "acc": 0.23, + "acc_stderr": 0.04229525846816505, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542126 + }, + "crows_pairs_english_sexual_orientation": { + "likelihood_difference": 4.742103494623656, + "likelihood_difference_stderr": 0.5813902425880072, + "pct_stereotype": 0.7526881720430108, + "pct_stereotype_stderr": 0.0449817218566707 + }, + "crows_pairs_french_age": { + "likelihood_difference": 5.097569444444445, + "likelihood_difference_stderr": 0.5161568677503977, + "pct_stereotype": 0.4888888888888889, + "pct_stereotype_stderr": 0.05298680599073449 + }, + "hendrycksTest-computer_security": { + "acc": 0.16, + "acc_stderr": 0.036845294917747094, + "acc_norm": 0.28, + "acc_norm_stderr": 0.04512608598542127 + }, + "hendrycksTest-high_school_us_history": { + "acc": 0.22549019607843138, + "acc_stderr": 0.029331162294251714, + "acc_norm": 0.2549019607843137, + "acc_norm_stderr": 0.030587591351604246 + }, + "hendrycksTest-high_school_european_history": { + "acc": 0.21212121212121213, + "acc_stderr": 0.03192271569548299, + "acc_norm": 0.3090909090909091, + "acc_norm_stderr": 0.036085410115739666 + }, + "crows_pairs_french_gender": { + "likelihood_difference": 4.377141744548287, + "likelihood_difference_stderr": 0.2263841012514179, + "pct_stereotype": 0.5077881619937694, + "pct_stereotype_stderr": 0.027947458769356347 + }, + "hendrycksTest-high_school_mathematics": { + "acc": 0.16296296296296298, + "acc_stderr": 0.02251856199768264, + "acc_norm": 0.23333333333333334, + "acc_norm_stderr": 0.02578787422095933 + }, + "crows_pairs_english_religion": { + "likelihood_difference": 3.8074324324324325, + "likelihood_difference_stderr": 0.44590802870655577, + "pct_stereotype": 0.6126126126126126, + "pct_stereotype_stderr": 0.0464482507235508 + }, + "hendrycksTest-global_facts": { + "acc": 0.23, + "acc_stderr": 0.04229525846816506, + "acc_norm": 0.23, + "acc_norm_stderr": 0.042295258468165065 + }, + "hendrycksTest-clinical_knowledge": { + "acc": 0.2528301886792453, + "acc_stderr": 0.026749899771241235, + "acc_norm": 0.30943396226415093, + "acc_norm_stderr": 0.028450154794118627 + }, + "hendrycksTest-anatomy": { + "acc": 0.2222222222222222, + "acc_stderr": 0.035914440841969694, + "acc_norm": 0.2222222222222222, + "acc_norm_stderr": 0.035914440841969694 + }, + "hendrycksTest-business_ethics": { + "acc": 0.31, + "acc_stderr": 0.04648231987117316, + "acc_norm": 0.25, + "acc_norm_stderr": 0.04351941398892446 + }, + "arc_challenge": { + "acc": 0.17235494880546076, + "acc_stderr": 0.011037113093461295, + "acc_norm": 0.22013651877133106, + "acc_norm_stderr": 0.01210812488346098 + }, + "hendrycksTest-college_computer_science": { + "acc": 0.24, + "acc_stderr": 0.04292346959909282, + "acc_norm": 0.24, + "acc_norm_stderr": 0.04292346959909282 + }, + "hendrycksTest-world_religions": { + "acc": 0.2046783625730994, + "acc_stderr": 0.030944459778533207, + "acc_norm": 0.2573099415204678, + "acc_norm_stderr": 0.03352799844161865 + }, + "hendrycksTest-human_aging": { + "acc": 0.29596412556053814, + "acc_stderr": 0.030636591348699796, + "acc_norm": 0.26905829596412556, + "acc_norm_stderr": 0.029763779406874972 + }, + "hendrycksTest-philosophy": { + "acc": 0.21543408360128619, + "acc_stderr": 0.023350225475471418, + "acc_norm": 0.2572347266881029, + "acc_norm_stderr": 0.024826171289250888 + }, + "hendrycksTest-high_school_government_and_politics": { + "acc": 0.23834196891191708, + "acc_stderr": 0.030748905363909892, + "acc_norm": 0.27979274611398963, + "acc_norm_stderr": 0.032396370467357036 + }, + "hendrycksTest-miscellaneous": { + "acc": 0.27330779054916987, + "acc_stderr": 0.01593668106262856, + "acc_norm": 0.25287356321839083, + "acc_norm_stderr": 0.015543377313719681 + }, + "hendrycksTest-security_studies": { + "acc": 0.2897959183673469, + "acc_stderr": 0.029043088683304342, + "acc_norm": 0.2530612244897959, + "acc_norm_stderr": 0.02783302387139968 + }, + "hendrycksTest-management": { + "acc": 0.18446601941747573, + "acc_stderr": 0.03840423627288276, + "acc_norm": 0.2815533980582524, + "acc_norm_stderr": 0.04453254836326468 + }, + "crows_pairs_english_gender": { + "likelihood_difference": 3.0462890625, + "likelihood_difference_stderr": 0.27914349499619723, + "pct_stereotype": 0.4875, + "pct_stereotype_stderr": 0.02798587585995666 + }, + "crows_pairs_english_autre": { + "likelihood_difference": 5.657670454545454, + "likelihood_difference_stderr": 1.600279703203965, + "pct_stereotype": 0.5454545454545454, + "pct_stereotype_stderr": 0.1574591643244434 + }, + "hendrycksTest-college_biology": { + "acc": 0.2777777777777778, + "acc_stderr": 0.037455547914624576, + "acc_norm": 0.3125, + "acc_norm_stderr": 0.038760854559127644 + }, + "hendrycksTest-medical_genetics": { + "acc": 0.26, + "acc_stderr": 0.04408440022768078, + "acc_norm": 0.34, + "acc_norm_stderr": 0.04760952285695235 + }, + "hendrycksTest-high_school_psychology": { + "acc": 0.26605504587155965, + "acc_stderr": 0.018946022322225604, + "acc_norm": 0.26055045871559634, + "acc_norm_stderr": 0.01881918203485007 + }, + "hendrycksTest-nutrition": { + "acc": 0.22549019607843138, + "acc_stderr": 0.023929155517351277, + "acc_norm": 0.34967320261437906, + "acc_norm_stderr": 0.027305308076274702 + }, + "hendrycksTest-jurisprudence": { + "acc": 0.23148148148148148, + "acc_stderr": 0.04077494709252626, + "acc_norm": 0.3425925925925926, + "acc_norm_stderr": 0.04587904741301811 + }, + "crows_pairs_french_nationality": { + "likelihood_difference": 8.008646245059289, + "likelihood_difference_stderr": 0.4304168992412896, + "pct_stereotype": 0.2924901185770751, + "pct_stereotype_stderr": 0.028656396908494263 + }, + "hendrycksTest-human_sexuality": { + "acc": 0.2900763358778626, + "acc_stderr": 0.03980066246467766, + "acc_norm": 0.22900763358778625, + "acc_norm_stderr": 0.036853466317118506 + }, + "hendrycksTest-high_school_biology": { + "acc": 0.27741935483870966, + "acc_stderr": 0.025470196835900055, + "acc_norm": 0.29354838709677417, + "acc_norm_stderr": 0.02590608702131929 + }, + "winogrande": { + "acc": 0.5082872928176796, + "acc_stderr": 0.014050555322824194 + }, + "crows_pairs_english_age": { + "likelihood_difference": 2.9198145604395602, + "likelihood_difference_stderr": 0.29304502937485644, + "pct_stereotype": 0.42857142857142855, + "pct_stereotype_stderr": 0.05216405309573015 + }, + "hendrycksTest-international_law": { + "acc": 0.15702479338842976, + "acc_stderr": 0.0332124484254713, + "acc_norm": 0.38016528925619836, + "acc_norm_stderr": 0.04431324501968431 + }, + "piqa": { + "acc": 0.5979325353645266, + "acc_stderr": 0.011439867127267533, + "acc_norm": 0.5843307943416758, + "acc_norm_stderr": 0.011498699770894792 + }, + "crows_pairs_english_physical_appearance": { + "likelihood_difference": 4.0234375, + "likelihood_difference_stderr": 0.43062333624877625, + "pct_stereotype": 0.6388888888888888, + "pct_stereotype_stderr": 0.0570038146170086 + }, + "hendrycksTest-professional_medicine": { + "acc": 0.2536764705882353, + "acc_stderr": 0.02643132987078952, + "acc_norm": 0.26838235294117646, + "acc_norm_stderr": 0.0269174812243772 + }, + "sciq": { + "acc": 0.642, + "acc_stderr": 0.01516792886540756, + "acc_norm": 0.554, + "acc_norm_stderr": 0.015726771166750357 + }, + "crows_pairs_french_disability": { + "likelihood_difference": 6.504734848484849, + "likelihood_difference_stderr": 0.682563745515591, + "pct_stereotype": 0.4696969696969697, + "pct_stereotype_stderr": 0.06190336468479955 + }, + "hendrycksTest-college_medicine": { + "acc": 0.23121387283236994, + "acc_stderr": 0.032147373020294696, + "acc_norm": 0.2947976878612717, + "acc_norm_stderr": 0.034765996075164785 + }, + "crows_pairs_english": { + "likelihood_difference": 3.6300033542039354, + "likelihood_difference_stderr": 0.10616730589284588, + "pct_stereotype": 0.5020870602265951, + "pct_stereotype_stderr": 0.012213192820312024 + }, + "crows_pairs_english_socioeconomic": { + "likelihood_difference": 3.774342105263158, + "likelihood_difference_stderr": 0.25861312687838434, + "pct_stereotype": 0.6526315789473685, + "pct_stereotype_stderr": 0.03463365347393427 + }, + "hendrycksTest-professional_psychology": { + "acc": 0.2581699346405229, + "acc_stderr": 0.017704531653250068, + "acc_norm": 0.24509803921568626, + "acc_norm_stderr": 0.017401816711427657 + }, + "hendrycksTest-high_school_physics": { + "acc": 0.23178807947019867, + "acc_stderr": 0.034454062719870525, + "acc_norm": 0.2582781456953642, + "acc_norm_stderr": 0.035737053147634576 + }, + "hendrycksTest-conceptual_physics": { + "acc": 0.28936170212765955, + "acc_stderr": 0.02964400657700962, + "acc_norm": 0.22127659574468084, + "acc_norm_stderr": 0.02713634960242406 + }, + "hendrycksTest-high_school_geography": { + "acc": 0.24242424242424243, + "acc_stderr": 0.030532892233932036, + "acc_norm": 0.25252525252525254, + "acc_norm_stderr": 0.030954055470365904 + }, + "hendrycksTest-college_physics": { + "acc": 0.13725490196078433, + "acc_stderr": 0.034240846698915216, + "acc_norm": 0.20588235294117646, + "acc_norm_stderr": 0.04023382273617747 + }, + "crows_pairs_french_race_color": { + "likelihood_difference": 4.933559782608696, + "likelihood_difference_stderr": 0.23732252395314296, + "pct_stereotype": 0.26956521739130435, + "pct_stereotype_stderr": 0.02071172670289539 + }, + "hendrycksTest-public_relations": { + "acc": 0.2636363636363636, + "acc_stderr": 0.04220224692971987, + "acc_norm": 0.17272727272727273, + "acc_norm_stderr": 0.03620691833929219 + }, + "hendrycksTest-sociology": { + "acc": 0.27860696517412936, + "acc_stderr": 0.031700561834973086, + "acc_norm": 0.2835820895522388, + "acc_norm_stderr": 0.031871875379197966 + }, + "hendrycksTest-formal_logic": { + "acc": 0.2857142857142857, + "acc_stderr": 0.04040610178208841, + "acc_norm": 0.2619047619047619, + "acc_norm_stderr": 0.03932537680392871 + }, + "hendrycksTest-high_school_microeconomics": { + "acc": 0.2773109243697479, + "acc_stderr": 0.029079374539480007, + "acc_norm": 0.3487394957983193, + "acc_norm_stderr": 0.030956636328566545 + }, + "hendrycksTest-electrical_engineering": { + "acc": 0.27586206896551724, + "acc_stderr": 0.037245636197746325, + "acc_norm": 0.30344827586206896, + "acc_norm_stderr": 0.038312260488503336 + }, + "hendrycksTest-college_mathematics": { + "acc": 0.2, + "acc_stderr": 0.040201512610368445, + "acc_norm": 0.3, + "acc_norm_stderr": 0.046056618647183814 + }, + "hendrycksTest-college_chemistry": { + "acc": 0.3, + "acc_stderr": 0.046056618647183814, + "acc_norm": 0.32, + "acc_norm_stderr": 0.046882617226215034 + }, + "hendrycksTest-us_foreign_policy": { + "acc": 0.26, + "acc_stderr": 0.04408440022768077, + "acc_norm": 0.36, + "acc_norm_stderr": 0.048241815132442176 + }, + "crows_pairs_english_race_color": { + "likelihood_difference": 3.545275590551181, + "likelihood_difference_stderr": 0.17976008372744548, + "pct_stereotype": 0.4094488188976378, + "pct_stereotype_stderr": 0.021838590402568178 + }, + "hendrycksTest-professional_accounting": { + "acc": 0.2730496453900709, + "acc_stderr": 0.02657786094330786, + "acc_norm": 0.24822695035460993, + "acc_norm_stderr": 0.02577001564429039 + }, + "hendrycksTest-moral_scenarios": { + "acc": 0.24804469273743016, + "acc_stderr": 0.014444157808261427, + "acc_norm": 0.2737430167597765, + "acc_norm_stderr": 0.014912413096372432 + }, + "lambada_openai": { + "ppl": 140.52328755411287, + "ppl_stderr": 6.002931469828659, + "acc": 0.21405006792159906, + "acc_stderr": 0.00571435475116112 + }, + "hendrycksTest-high_school_macroeconomics": { + "acc": 0.2282051282051282, + "acc_stderr": 0.021278393863586282, + "acc_norm": 0.28205128205128205, + "acc_norm_stderr": 0.0228158130988966 + }, + "crows_pairs_french": { + "likelihood_difference": 5.462297816040548, + "likelihood_difference_stderr": 0.1326279922858767, + "pct_stereotype": 0.43231961836613, + "pct_stereotype_stderr": 0.012100892636108567 + }, + "crows_pairs_english_nationality": { + "likelihood_difference": 3.5159143518518516, + "likelihood_difference_stderr": 0.2836003217082339, + "pct_stereotype": 0.39351851851851855, + "pct_stereotype_stderr": 0.03331747876370312 + } + }, + "versions": { + "hendrycksTest-high_school_chemistry": 0, + "crows_pairs_english_disability": 0, + "hendrycksTest-marketing": 0, + "hendrycksTest-moral_disputes": 0, + "crows_pairs_french_physical_appearance": 0, + "hendrycksTest-astronomy": 0, + "hendrycksTest-professional_law": 0, + "hendrycksTest-high_school_world_history": 0, + "hendrycksTest-elementary_mathematics": 0, + "crows_pairs_french_autre": 0, + "hendrycksTest-high_school_computer_science": 0, + "hendrycksTest-logical_fallacies": 0, + "hendrycksTest-prehistory": 0, + "hendrycksTest-machine_learning": 0, + "wsc": 0, + "hendrycksTest-econometrics": 0, + "hendrycksTest-high_school_statistics": 0, + "crows_pairs_french_religion": 0, + "crows_pairs_french_sexual_orientation": 0, + "crows_pairs_french_socioeconomic": 0, + "hendrycksTest-virology": 0, + "arc_easy": 0, + "logiqa": 0, + "hendrycksTest-abstract_algebra": 0, + "crows_pairs_english_sexual_orientation": 0, + "crows_pairs_french_age": 0, + "hendrycksTest-computer_security": 0, + "hendrycksTest-high_school_us_history": 0, + "hendrycksTest-high_school_european_history": 0, + "crows_pairs_french_gender": 0, + "hendrycksTest-high_school_mathematics": 0, + "crows_pairs_english_religion": 0, + "hendrycksTest-global_facts": 0, + "hendrycksTest-clinical_knowledge": 0, + "hendrycksTest-anatomy": 0, + "hendrycksTest-business_ethics": 0, + "arc_challenge": 0, + "hendrycksTest-college_computer_science": 0, + "hendrycksTest-world_religions": 0, + "hendrycksTest-human_aging": 0, + "hendrycksTest-philosophy": 0, + "hendrycksTest-high_school_government_and_politics": 0, + "hendrycksTest-miscellaneous": 0, + "hendrycksTest-security_studies": 0, + "hendrycksTest-management": 0, + "crows_pairs_english_gender": 0, + "crows_pairs_english_autre": 0, + "hendrycksTest-college_biology": 0, + "hendrycksTest-medical_genetics": 0, + "hendrycksTest-high_school_psychology": 0, + "hendrycksTest-nutrition": 0, + "hendrycksTest-jurisprudence": 0, + "crows_pairs_french_nationality": 0, + "hendrycksTest-human_sexuality": 0, + "hendrycksTest-high_school_biology": 0, + "winogrande": 0, + "crows_pairs_english_age": 0, + "hendrycksTest-international_law": 0, + "piqa": 0, + "crows_pairs_english_physical_appearance": 0, + "hendrycksTest-professional_medicine": 0, + "sciq": 0, + "crows_pairs_french_disability": 0, + "hendrycksTest-college_medicine": 0, + "crows_pairs_english": 0, + "crows_pairs_english_socioeconomic": 0, + "hendrycksTest-professional_psychology": 0, + "hendrycksTest-high_school_physics": 0, + "hendrycksTest-conceptual_physics": 0, + "hendrycksTest-high_school_geography": 0, + "hendrycksTest-college_physics": 0, + "crows_pairs_french_race_color": 0, + "hendrycksTest-public_relations": 0, + "hendrycksTest-sociology": 0, + "hendrycksTest-formal_logic": 0, + "hendrycksTest-high_school_microeconomics": 0, + "hendrycksTest-electrical_engineering": 0, + "hendrycksTest-college_mathematics": 0, + "hendrycksTest-college_chemistry": 0, + "hendrycksTest-us_foreign_policy": 0, + "crows_pairs_english_race_color": 0, + "hendrycksTest-professional_accounting": 0, + "hendrycksTest-moral_scenarios": 0, + "lambada_openai": 0, + "hendrycksTest-high_school_macroeconomics": 0, + "crows_pairs_french": 0, + "crows_pairs_english_nationality": 0 + }, + "config": { + "model": "hf-causal", + "model_args": "pretrained=EleutherAI/pythia-v1.1-70m,revision=step93000", + "num_fewshot": 0, + "batch_size": 16, + "device": "cuda:1", + "no_cache": true, + "limit": null, + "bootstrap_iters": 100000, + "description_dict": {} + } +} \ No newline at end of file diff --git a/data/checkpoint_eval/pythia-70m_checkpoint_comparison.json b/data/checkpoint_eval/pythia-70m_checkpoint_comparison.json new file mode 100644 index 0000000000000000000000000000000000000000..cdc63b353b6901b16e821b4c5eadf10eebb11860 --- /dev/null +++ b/data/checkpoint_eval/pythia-70m_checkpoint_comparison.json @@ -0,0 +1,48 @@ +{ + "step8000": { + "revision": "step8000", + "expected_gamma": 1.064, + "gamma_obs": 1.0266835966009398, + "r2": 0.9820928579517015, + "attn_means_per_distance": [ + 0.013771302501360575, + 0.008232926908466548, + 0.00256258381737603, + 0.002347815906008085, + 0.0010662785420815148, + 0.0002845022413465712, + 0.00010495591494772167, + 7.333743075529732e-05 + ], + "passkey_accuracy": { + "256": 0.05, + "512": 0.2, + "1024": 0.05, + "2048": 0.0 + }, + "note_public_benchmarks": "LAMBADA/HellaSwag/PPL available in combined_training_dynamics.json from EleutherAI public evals" + }, + "step143000": { + "revision": "step143000", + "expected_gamma": 0.781, + "gamma_obs": NaN, + "r2": NaN, + "attn_means_per_distance": [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + "passkey_accuracy": { + "256": 0.0, + "512": 0.0, + "1024": 0.0, + "2048": 0.0 + }, + "note_public_benchmarks": "LAMBADA/HellaSwag/PPL available in combined_training_dynamics.json from EleutherAI public evals" + } +} \ No newline at end of file diff --git a/data/cloud/gqa_rc_results.json b/data/cloud/gqa_rc_results.json new file mode 100644 index 0000000000000000000000000000000000000000..34f43a8adae367f120c252c40c2434ab21e864aa --- /dev/null +++ b/data/cloud/gqa_rc_results.json @@ -0,0 +1,270 @@ +{ + "results": [ + { + "label": "Qwen2.5-0.5B", + "hf_id": "Qwen/Qwen2.5-0.5B", + "n_kv": 2, + "d_head": 64, + "n_params_M": 490, + "theta": 1000000, + "T_train": 32768, + "R_c": 17.07756720897567, + "random": { + "gamma": 0.9062338770446412, + "r2": 0.9909270565050566, + "means": { + "10": 0.00933715307641597, + "20": 0.008325269828949655, + "30": 0.0052897306567146665, + "50": 0.0035934632803712573, + "100": 0.0014672519408521199, + "200": 0.0010016655212356932, + "500": 0.0004682919560443787, + "1000": 0.00020250596815631503 + } + }, + "text": { + "gamma": 0.8614455324492792, + "r2": 0.9644449017304708, + "means": { + "10": 0.008442668694825399, + "20": 0.00615490938226382, + "30": 0.004094105288386345, + "50": 0.002889665500039146, + "100": 0.0011341503723746255, + "200": 0.0006860575541144325, + "500": 0.000253526685493333, + "1000": 0.0002539449149654025 + } + }, + "delta_gamma": 0.04478834459536196, + "verdict": "neutral" + }, + { + "label": "Qwen2.5-1.5B", + "hf_id": "Qwen/Qwen2.5-1.5B", + "n_kv": 2, + "d_head": 128, + "n_params_M": 1540, + "theta": 1000000, + "T_train": 32768, + "R_c": 3.085057430752342, + "random": { + "gamma": 0.980402153144676, + "r2": 0.9594453213557013, + "means": { + "10": 0.01075944620832533, + "20": 0.006380008729985905, + "30": 0.003982916441592542, + "50": 0.0022768721139679354, + "100": 0.0010736816094486287, + "200": 0.0008009741012054153, + "500": 0.00015360812346140544, + "1000": 0.000157832403977712 + } + }, + "text": { + "gamma": 1.0634506863606663, + "r2": 0.9769510380449103, + "means": { + "10": 0.010448309971608, + "20": 0.006884462779123603, + "30": 0.0044515742664843535, + "50": 0.002194346263345364, + "100": 0.0008933738613805027, + "200": 0.0004605254583191453, + "500": 0.00014063805924003966, + "1000": 0.00012315947113307655 + } + }, + "delta_gamma": -0.08304853321599026, + "verdict": "neutral" + }, + { + "label": "SmolLM2-1.7B", + "hf_id": "HuggingFaceTB/SmolLM2-1.7B", + "n_kv": 32, + "d_head": 64, + "n_params_M": 1700, + "theta": 130000, + "T_train": 8192, + "R_c": 0.47905044355233856, + "random": { + "gamma": 0.8662591994464454, + "r2": 0.9970100032742922, + "means": { + "10": 0.00644491306040436, + "20": 0.0052320567176987725, + "30": 0.0030720023019239306, + "50": 0.0020971989150469503, + "100": 0.0012476958287879825, + "200": 0.0006285492129003009, + "500": 0.00025838499888777735, + "1000": 0.00016099023167043925 + } + }, + "text": { + "gamma": 0.8459469449319719, + "r2": 0.9628408330570999, + "means": { + "10": 0.005857278377128144, + "20": 0.0038359998057906825, + "30": 0.0027658051640416185, + "50": 0.0016047325575103363, + "100": 0.0008706180502971013, + "200": 0.0006290887234111627, + "500": 0.0003532431973144412, + "1000": 0.00010335376486182212 + } + }, + "delta_gamma": 0.020312254514473493, + "verdict": "neutral" + }, + { + "label": "Phi-3.5-mini", + "hf_id": "microsoft/Phi-3.5-mini-instruct", + "n_kv": 32, + "d_head": 96, + "n_params_M": 3820, + "theta": 10000, + "T_train": 131072, + "R_c": 0.48685674820547287, + "error": "measure: type object 'DynamicCache' has no attribute 'from_legacy_cache'" + }, + { + "label": "Qwen2.5-3B", + "hf_id": "Qwen/Qwen2.5-3B", + "n_kv": 2, + "d_head": 128, + "n_params_M": 3090, + "theta": 1000000, + "T_train": 32768, + "R_c": 2.0342189993534796, + "random": { + "gamma": 0.7464802030156852, + "r2": 0.9879267676037468, + "means": { + "10": 0.009178792749428087, + "20": 0.006702715104652776, + "30": 0.004231071588065889, + "50": 0.003230642717745569, + "100": 0.0015271495696571138, + "200": 0.0009070214380820593, + "500": 0.00047202539319793384, + "1000": 0.000346068793700801 + } + }, + "text": { + "gamma": 0.799209027274833, + "r2": 0.927322958064882, + "means": { + "10": 0.008374775254891978, + "20": 0.004851305995964342, + "30": 0.0030537243146035405, + "50": 0.0022043935333689053, + "100": 0.0010392124433484342, + "200": 0.0005058742376665274, + "500": 0.00019562873782383072, + "1000": 0.0002686874713334772 + } + }, + "delta_gamma": -0.05272882425914782, + "verdict": "neutral" + }, + { + "label": "Llama-3.2-3B", + "hf_id": "meta-llama/Llama-3.2-3B", + "n_kv": 8, + "d_head": 128, + "n_params_M": 3210, + "theta": 500000, + "T_train": 131072, + "R_c": 1.331335711921419, + "random": { + "gamma": 0.678947152194417, + "r2": 0.9423162919570118, + "means": { + "10": 0.0053288231985200015, + "20": 0.003921971014212994, + "30": 0.0029104793018528393, + "50": 0.002017381762464841, + "100": 0.0014156586463962282, + "200": 0.0010305073467038928, + "500": 0.00029545511518205917, + "1000": 0.00033367779638086045 + } + }, + "text": { + "gamma": 0.5942645260566438, + "r2": 0.9633619728314922, + "means": { + "10": 0.006716178983804726, + "20": 0.004247029634813468, + "30": 0.0026757050908747175, + "50": 0.0018435514008715039, + "100": 0.0012482281100182305, + "200": 0.0007410081138923054, + "500": 0.00036949885388215385, + "1000": 0.00039038256520316714 + } + }, + "delta_gamma": 0.08468262613777322, + "verdict": "neutral" + }, + { + "label": "Yi-6B", + "hf_id": "01-ai/Yi-6B", + "n_kv": 4, + "d_head": 128, + "n_params_M": 6060, + "theta": 5000000, + "T_train": 4096, + "R_c": 1.2751049391099323, + "random": { + "gamma": 0.9664360975501075, + "r2": 0.9795433573586503, + "means": { + "10": 0.007488797352125403, + "20": 0.006183944555632479, + "30": 0.0037867348757026774, + "50": 0.0034175124804460212, + "100": 0.0016160887095566067, + "200": 0.0010107121610053581, + "500": 0.00031756698555682303, + "1000": 0.00014130359336604669 + } + }, + "text": { + "gamma": 0.841961271334462, + "r2": 0.9822390595591267, + "means": { + "10": 0.006165254368618207, + "20": 0.004001568540134031, + "30": 0.0020899126197446096, + "50": 0.001873674674262702, + "100": 0.0009678956000712446, + "200": 0.0006449051997040556, + "500": 0.00022300777077084887, + "1000": 0.00012313241315040636 + } + }, + "delta_gamma": 0.12447482621564554, + "verdict": "pre-IH" + }, + { + "label": "Qwen2.5-7B", + "hf_id": "Qwen/Qwen2.5-7B", + "n_kv": 4, + "d_head": 128, + "n_params_M": 7610, + "theta": 1000000, + "T_train": 131072, + "R_c": 1.1765191580975, + "error": "measure: CUDA out of memory. Tried to allocate 48.00 MiB. GPU 0 has a total capacity of 14.56 GiB of which 33.81 MiB is free. Including non-PyTorch memory, this process has 14.53 GiB memory in use. Of the allocated memory 14.37 GiB is allocated by PyTorch, and 27.00 MiB is reserved by PyTorch but unallocated. If reserved but unallocated memory is large try setting PYTORCH_ALLOC_CONF=expandable_segments:True to avoid fragmentation. See documentation for Memory Management (https://pytorch.org/docs/stable/notes/cuda.html#environment-variables)" + } + ], + "elapsed_min": 62.35337897141775, + "n_completed": 6, + "panel_size": 12, + "incomplete_reason": "disk space limit" +} \ No newline at end of file diff --git a/data/cloud/round4_combined_results.json b/data/cloud/round4_combined_results.json new file mode 100644 index 0000000000000000000000000000000000000000..9d2cfb3cfd95e7161704b920a5a4b4364e4c9d66 --- /dev/null +++ b/data/cloud/round4_combined_results.json @@ -0,0 +1,81 @@ +{ + "phase_AB_rc": [ + { + "label": "Mistral-7B (4bit)", + "hf_id": "mistralai/Mistral-7B-v0.1", + "phase": "A", + "n_kv": 8, + "d_head": 128, + "n_params_M": 7240, + "theta": 10000, + "T_train": 32768, + "R_c": 0.9574147216280355, + "error": "load: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`" + }, + { + "label": "Gemma-2-9b (4bit)", + "hf_id": "google/gemma-2-9b", + "phase": "A", + "n_kv": 16, + "d_head": 256, + "n_params_M": 9240, + "theta": 10000, + "T_train": 8192, + "R_c": 0.8830371105651076, + "error": "load: You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/google/gemma-2-9b.\n401 Client Error. (Request ID: Root=1-69ecc931-748a97af79a21cbd0472673c;838debda-8ccf-406b-a3c9-b7e1f43062b3)\n\nCannot access gated repo for url https://huggingface.co/google/gemma-2-9b/resolve/main/config.json.\nAccess to model google/gemma-2-9b is restricted. You must have access to it and be authenticated to access it. Please log in." + }, + { + "label": "OLMo-7B (4bit)", + "hf_id": "allenai/OLMo-7B-hf", + "phase": "B", + "n_kv": 32, + "d_head": 128, + "n_params_M": 6890, + "theta": 10000, + "T_train": 2048, + "R_c": 0.48704080868743954, + "error": "load: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`" + }, + { + "label": "Falcon-7B (4bit)", + "hf_id": "tiiuae/falcon-7b", + "phase": "B", + "n_kv": 1, + "d_head": 64, + "n_params_M": 6920, + "theta": 10000, + "T_train": 2048, + "R_c": 1.4588955689285732, + "error": "load: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`" + }, + { + "label": "pythia-6.9b (4bit)", + "hf_id": "EleutherAI/pythia-6.9b", + "phase": "B", + "n_kv": 32, + "d_head": 128, + "n_params_M": 6900, + "theta": 10000, + "T_train": 2048, + "R_c": 0.4867927697741792, + "error": "load: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`" + } + ], + "phase_C_multifractal": [ + { + "label": "Llama-3-8B (4bit)", + "hf_id": "meta-llama/Meta-Llama-3-8B", + "phase": "C", + "error": "load: You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/meta-llama/Meta-Llama-3-8B.\n401 Client Error. (Request ID: Root=1-69ecc936-20895f6374a7bc1b2e481224;f56b13e3-d463-4c02-a912-bb4a7de5e9d6)\n\nCannot access gated repo for url https://huggingface.co/meta-llama/Meta-Llama-3-8B/resolve/main/config.json.\nAccess to model meta-llama/Meta-Llama-3-8B is restricted. You must have access to it and be authenticated to access it. Please log in." + }, + { + "label": "Mistral-7B (4bit)", + "hf_id": "mistralai/Mistral-7B-v0.1", + "phase": "C", + "error": "load: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`" + } + ], + "elapsed_min": 0.17913634777069093, + "n_completed_rc": 0, + "n_completed_mf": 0 +} \ No newline at end of file diff --git a/data/cloud/round4_combined_results_01.json b/data/cloud/round4_combined_results_01.json new file mode 100644 index 0000000000000000000000000000000000000000..d9ab48c82540416cf0b3a3a19eec4169ac612d93 --- /dev/null +++ b/data/cloud/round4_combined_results_01.json @@ -0,0 +1,81 @@ +{ + "phase_AB_rc": [ + { + "label": "Mistral-7B (4bit)", + "hf_id": "mistralai/Mistral-7B-v0.1", + "phase": "A", + "n_kv": 8, + "d_head": 128, + "n_params_M": 7240, + "theta": 10000, + "T_train": 32768, + "R_c": 0.9574147216280355, + "error": "load: Illegal header value b'Bearer '" + }, + { + "label": "Qwen2.5-7B (4bit)", + "hf_id": "Qwen/Qwen2.5-7B", + "phase": "A", + "n_kv": 4, + "d_head": 128, + "n_params_M": 7610, + "theta": 1000000, + "T_train": 131072, + "R_c": 1.1765191580975, + "error": "load: Illegal header value b'Bearer '" + }, + { + "label": "OLMo-7B (4bit)", + "hf_id": "allenai/OLMo-7B-hf", + "phase": "B", + "n_kv": 32, + "d_head": 128, + "n_params_M": 6890, + "theta": 10000, + "T_train": 2048, + "R_c": 0.48704080868743954, + "error": "load: Illegal header value b'Bearer '" + }, + { + "label": "Falcon-7B (4bit)", + "hf_id": "tiiuae/falcon-7b", + "phase": "B", + "n_kv": 1, + "d_head": 64, + "n_params_M": 6920, + "theta": 10000, + "T_train": 2048, + "R_c": 1.4588955689285732, + "error": "load: Illegal header value b'Bearer '" + }, + { + "label": "pythia-6.9b (4bit)", + "hf_id": "EleutherAI/pythia-6.9b", + "phase": "B", + "n_kv": 32, + "d_head": 128, + "n_params_M": 6900, + "theta": 10000, + "T_train": 2048, + "R_c": 0.4867927697741792, + "error": "load: Illegal header value b'Bearer '" + } + ], + "phase_C_multifractal": [ + { + "label": "Llama-3-8B (4bit, Nous mirror)", + "hf_id": "NousResearch/Meta-Llama-3-8B", + "phase": "C", + "error": "load: Illegal header value b'Bearer '" + }, + { + "label": "Mistral-7B (4bit)", + "hf_id": "mistralai/Mistral-7B-v0.1", + "phase": "C", + "error": "load: Illegal header value b'Bearer '" + } + ], + "elapsed_min": 0.029302664597829185, + "n_completed_rc": 0, + "n_completed_mf": 0 +} \ No newline at end of file diff --git a/data/cloud/round4_combined_results_02.json b/data/cloud/round4_combined_results_02.json new file mode 100644 index 0000000000000000000000000000000000000000..e5da034b764d01c8f6c0f3c4ae88da70c2709a23 --- /dev/null +++ b/data/cloud/round4_combined_results_02.json @@ -0,0 +1,104 @@ +{ + "phase_AB_rc": [ + { + "label": "Mistral-7B (4bit)", + "hf_id": "mistralai/Mistral-7B-v0.1", + "phase": "A", + "n_kv": 8, + "d_head": 128, + "n_params_M": 7240, + "theta": 10000, + "T_train": 32768, + "R_c": 0.9574147216280355, + "random": { + "gamma": 0.8341837934571041, + "r2": 0.677964594790021, + "means": { + "10": 0.007097774606736493, + "20": 0.005262479490847909, + "30": 0.004260748066699307, + "50": 0.003949870442265819, + "100": 0.013004653203142879, + "200": 0.001001053715207263, + "500": 0.0006962497123834055, + "1000": 0.0003346679925857643 + } + }, + "text": { + "gamma": 0.981381766010483, + "r2": 0.9904316492608461, + "means": { + "10": 0.00840350326976477, + "20": 0.004405914865783416, + "30": 0.0032953370028644714, + "50": 0.0023159317179761276, + "100": 0.0011866624017125105, + "200": 0.0006131482139687705, + "500": 0.0002772645528344242, + "1000": 9.826288272485328e-05 + } + }, + "delta_gamma": -0.14719797255337885, + "verdict": "post-IH" + }, + { + "label": "Qwen2.5-7B (4bit)", + "hf_id": "Qwen/Qwen2.5-7B", + "phase": "A", + "n_kv": 4, + "d_head": 128, + "n_params_M": 7610, + "theta": 1000000, + "T_train": 131072, + "R_c": 1.1765191580975, + "error": "load: No module named 'transformers.models.audioflamingo3'" + }, + { + "label": "OLMo-7B (4bit)", + "hf_id": "allenai/OLMo-7B-hf", + "phase": "B", + "n_kv": 32, + "d_head": 128, + "n_params_M": 6890, + "theta": 10000, + "T_train": 2048, + "R_c": 0.48704080868743954, + "error": "load: cannot import name 'FLAX_WEIGHTS_NAME' from 'transformers.utils' (/usr/local/lib/python3.12/dist-packages/transformers/utils/__init__.py)" + }, + { + "label": "Falcon-7B (4bit)", + "hf_id": "tiiuae/falcon-7b", + "phase": "B", + "n_kv": 1, + "d_head": 64, + "n_params_M": 6920, + "theta": 10000, + "T_train": 2048, + "R_c": 1.4588955689285732, + "error": "measure: 'FalconModel' object has no attribute 'get_head_mask'" + }, + { + "label": "pythia-6.9b (4bit)", + "hf_id": "EleutherAI/pythia-6.9b", + "phase": "B", + "n_kv": 32, + "d_head": 128, + "n_params_M": 6900, + "theta": 10000, + "T_train": 2048, + "R_c": 0.4867927697741792, + "error": "load: cannot import name 'FLAX_WEIGHTS_NAME' from 'transformers.utils' (/usr/local/lib/python3.12/dist-packages/transformers/utils/__init__.py)" + } + ], + "phase_C_multifractal": [ + { + "label": "Llama-3-8B (4bit, Nous mirror)", + "hf_id": "NousResearch/Meta-Llama-3-8B", + "phase": "C", + "error": "load: 'default'" + } + ], + "elapsed_min": 36.784542791048686, + "n_completed_rc": 1, + "n_completed_mf": 0 +} \ No newline at end of file diff --git a/data/compress_validation/compression_validation.json b/data/compress_validation/compression_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..aa7b8b44f292ba22ae3a6a15bf3abb730408292e --- /dev/null +++ b/data/compress_validation/compression_validation.json @@ -0,0 +1,484 @@ +[ + { + "gamma": 0.6276459084140061, + "log_A": -2.864020841938658, + "R2": 0.977314, + "theta": 10000, + "corpus": "mongo", + "model": "google/gemma-2-9b-it", + "attn_empirical": { + "10": 0.017953182898410077, + "20": 0.010547844236328577, + "30": 0.007733995182853605, + "50": 0.004784861977936493, + "100": 0.0028697990643559023, + "200": 0.0017543701516721437, + "500": 0.0011422816235285103, + "1000": 0.000969841086035659, + "2000": 0.0004331948929355652 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "google--gemma-2-9b-it_mongo", + "mape": 11.143181244037981, + "d90_th": 1524, + "d90_em": 1529, + "err_pct": 0.3270111183780248, + "dH_90": -0.02993155667320758, + "ratio90": 0.762 + }, + { + "gamma": 0.6741618914822415, + "log_A": -3.179715570803609, + "R2": 0.999287, + "theta": 10000, + "corpus": "mongo", + "model": "EleutherAI/pythia-2.8b", + "attn_empirical": { + "10": 0.008272597978745277, + "20": 0.005688730217193047, + "30": 0.004169839709583256, + "50": 0.003013845277988973, + "100": 0.0018553994908062225 + }, + "distances_fit": [ + 30, + 50, + 100 + ], + "stem": "EleutherAI--pythia-2.8b_mongo", + "mape": 0.8329977233081147, + "d90_th": 1476, + "d90_em": 81, + "err_pct": 1722.2222222222222, + "dH_90": -0.033000193771156476, + "ratio90": 0.738 + }, + { + "gamma": 0.7476017873166874, + "log_A": -2.391511197086578, + "R2": 0.984269, + "theta": 10000, + "corpus": "mongo", + "model": "EleutherAI/pythia-70m", + "attn_empirical": { + "10": 0.015389821239643628, + "20": 0.011693157334811985, + "30": 0.007082567571972807, + "50": 0.0054180885271893605, + "100": 0.002821484198017667, + "200": 0.0018563934957556841, + "500": 0.0007652510016729745, + "1000": 0.0004260873921642391, + "2000": 0.00039341128339098454 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "EleutherAI--pythia-70m_mongo", + "mape": 11.302149727500431, + "d90_th": 1383, + "d90_em": 1590, + "err_pct": 13.018867924528301, + "dH_90": -0.03897260917359724, + "ratio90": 0.6915 + }, + { + "gamma": 0.8266242679750889, + "log_A": -2.9325874169558817, + "R2": 0.993628, + "theta": 10000, + "corpus": "random", + "model": "meta-llama/Llama-2-7b-hf", + "attn_empirical": { + "10": 0.005234783389605581, + "20": 0.0032586656397001613, + "30": 0.0027830680216559107, + "50": 0.0020982515016415466, + "100": 0.0013744194064444551, + "200": 0.0006745984775221183, + "500": 0.0003505304501029766, + "1000": 0.00016617169836536053, + "2000": 9.237132345636685e-05 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "meta-llama--Llama-2-7b-hf_random", + "mape": 7.805256961731357, + "d90_th": 1254, + "d90_em": 1381, + "err_pct": 9.196234612599566, + "dH_90": -0.04731642837652837, + "ratio90": 0.627 + }, + { + "gamma": 0.8296009929924347, + "log_A": -2.376214984270495, + "R2": 0.996923, + "theta": 10000, + "corpus": "random", + "model": "mistralai/Mistral-7B-v0.1", + "attn_empirical": { + "10": 0.009985529906633829, + "20": 0.0060750810507064065, + "30": 0.005252508784696045, + "50": 0.0036803160721643104, + "100": 0.0022635483171325175, + "200": 0.0010285094767136292, + "500": 0.0005741028257438707, + "1000": 0.00029642416384174594, + "2000": 0.0001669441466219723 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "mistralai--Mistral-7B-v0.1_random", + "mape": 5.478842947828659, + "d90_th": 1248, + "d90_em": 1406, + "err_pct": 11.2375533428165, + "dH_90": -0.04770621251295322, + "ratio90": 0.624 + }, + { + "gamma": 0.9311078627189842, + "log_A": -2.350543685121484, + "R2": 0.983104, + "theta": 10000, + "corpus": "mongo", + "model": "EleutherAI/pythia-1b", + "attn_empirical": { + "10": 0.010958388174573582, + "20": 0.006365462491909664, + "30": 0.003956688532812728, + "50": 0.0027740057195640272, + "100": 0.0014326052833348512, + "200": 0.0006671193842258718, + "500": 0.00020134633510477014, + "1000": 0.00015050364037354788, + "2000": 0.00010232122304538885 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "EleutherAI--pythia-1b_mongo", + "mape": 13.09086093814886, + "d90_th": 1028, + "d90_em": 1370, + "err_pct": 24.963503649635037, + "dH_90": -0.062104239203459546, + "ratio90": 0.514 + }, + { + "gamma": 0.9966953735480816, + "log_A": -2.1584093095473813, + "R2": 0.993942, + "theta": 1000000, + "corpus": "mongo", + "model": "Qwen/Qwen2.5-7B", + "attn_empirical": { + "10": 0.01053441942591841, + "20": 0.005741046461561281, + "30": 0.003944915988637755, + "50": 0.002418298812634829, + "100": 0.0012223406427882666, + "200": 0.0004989972161840544, + "500": 0.0002234310967226823, + "1000": 0.00014713535652036727, + "2000": 5.419184616888136e-05 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "Qwen--Qwen2.5-7B_mongo", + "mape": 8.69563111859062, + "d90_th": 856, + "d90_em": 990, + "err_pct": 13.535353535353536, + "dH_90": -0.0008081142096023077, + "ratio90": 0.428 + }, + { + "gamma": 1.003714187534367, + "log_A": -1.6477058895472607, + "R2": 0.977698, + "theta": null, + "corpus": "random", + "model": "EleutherAI/pythia-14m", + "attn_empirical": { + "10": 0.010373582980052257, + "20": 0.006793614017466704, + "30": 0.005060315049162858, + "50": 0.0042915507254656406, + "100": 0.0019918971036935952, + "200": 0.0011028067811053512, + "500": 0.00045950785397306186, + "1000": 0.00012169795030211442, + "2000": 0.00010627185478203357 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "EleutherAI--pythia-14m_random", + "mape": 20.067563393050918, + "d90_th": 836, + "d90_em": 1225, + "err_pct": 31.755102040816325, + "dH_90": -0.07484146013435433, + "ratio90": 0.418 + }, + { + "gamma": 1.0171452847779678, + "log_A": -1.8268598516498322, + "R2": 0.981723, + "theta": 10000, + "corpus": "random", + "model": "EleutherAI/pythia-160m", + "attn_empirical": { + "10": 0.010829462509912749, + "20": 0.009873490159192847, + "30": 0.003808915811306279, + "50": 0.003134424632622136, + "100": 0.001589946753811091, + "200": 0.0009230124831406607, + "500": 0.0003505013234017598, + "1000": 0.00015388178632242167, + "2000": 5.142585230411594e-05 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "EleutherAI--pythia-160m_random", + "mape": 17.944445910277434, + "d90_th": 799, + "d90_em": 870, + "err_pct": 8.160919540229886, + "dH_90": -0.07731478366925829, + "ratio90": 0.3995 + }, + { + "gamma": 1.0218530106365162, + "log_A": -1.7669627940483377, + "R2": 0.981594, + "theta": 10000, + "corpus": "mongo", + "model": "EleutherAI/pythia-410m", + "attn_empirical": { + "10": 0.010637531184022211, + "20": 0.007524669199354119, + "30": 0.004181321816156721, + "50": 0.0032434275430083897, + "100": 0.001533225679288282, + "200": 0.000936404794475594, + "500": 0.0003803397886238397, + "1000": 0.0001626184749986553, + "2000": 5.13333430424407e-05 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "EleutherAI--pythia-410m_mongo", + "mape": 17.34396231950948, + "d90_th": 785, + "d90_em": 881, + "err_pct": 10.896708286038592, + "dH_90": -0.07825223337361463, + "ratio90": 0.3925 + }, + { + "gamma": 1.0454762537473639, + "log_A": -2.4338207488763257, + "R2": 0.997461, + "theta": 500000, + "corpus": "mongo", + "model": "meta-llama/Meta-Llama-3-8B", + "attn_empirical": { + "10": 0.0059040391059695845, + "20": 0.0036818826782594742, + "30": 0.0025728944933184213, + "50": 0.0014714971260077114, + "100": 0.0006953646930115712, + "200": 0.00037121372909395075, + "500": 0.00011184491504738818, + "1000": 6.471107897798195e-05, + "2000": 3.34952792859945e-05 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "meta-llama--Meta-Llama-3-8B_mongo", + "mape": 5.5592357782201764, + "d90_th": 718, + "d90_em": 1037, + "err_pct": 30.76181292189007, + "dH_90": -0.0018095444887590362, + "ratio90": 0.359 + }, + { + "gamma": 1.060750419523944, + "log_A": -2.143867119472637, + "R2": 0.99869, + "theta": 10000, + "corpus": "mongo", + "model": "mistralai/Mistral-7B-v0.1", + "attn_empirical": { + "10": 0.006733735190031843, + "20": 0.004026857788234742, + "30": 0.0030662473954726015, + "50": 0.0019213843246042315, + "100": 0.0009038505519533323, + "200": 0.00039353982273799675, + "500": 0.0001694695357905908, + "1000": 8.260491097138987e-05, + "2000": 3.443458402115438e-05 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "mistralai--Mistral-7B-v0.1_mongo", + "mape": 5.214559494114403, + "d90_th": 674, + "d90_em": 947, + "err_pct": 28.827877507919748, + "dH_90": -0.08571614105160623, + "ratio90": 0.337 + }, + { + "gamma": 1.1347958464287666, + "log_A": -0.9640958037685541, + "R2": 0.976472, + "theta": 10000, + "corpus": "random", + "model": "google/gemma-2-9b-it", + "attn_empirical": { + "10": 0.01131202671935575, + "20": 0.007342287618666887, + "30": 0.005608218560616176, + "50": 0.004600468463678327, + "100": 0.0025041243884091576, + "200": 0.0011723479800275526, + "500": 0.00037003465085743104, + "1000": 0.0001858278461845152, + "2000": 4.510536613654242e-05 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "google--gemma-2-9b-it_random", + "mape": 23.66528743924567, + "d90_th": 469, + "d90_em": 798, + "err_pct": 41.228070175438596, + "dH_90": -0.09964902083307159, + "ratio90": 0.2345 + }, + { + "gamma": 1.2350013988825523, + "log_A": -0.8481173688844952, + "R2": 0.973742, + "theta": 10000, + "corpus": "mongo", + "model": "EleutherAI/pythia-31m", + "attn_empirical": { + "10": 0.015075270254164932, + "20": 0.007979718983923603, + "30": 0.004943025353131816, + "50": 0.003909375242526746, + "100": 0.001335802334417369, + "200": 0.0006274800656319712, + "500": 0.0003229697247034993, + "1000": 0.00010229767169578944, + "2000": 2.207666448479115e-05 + }, + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "stem": "EleutherAI--pythia-31m_mongo", + "mape": 24.468814041997238, + "d90_th": 246, + "d90_em": 572, + "err_pct": 56.993006993006986, + "dH_90": -0.11502902082432076, + "ratio90": 0.123 + } +] \ No newline at end of file diff --git a/data/dft_weights/Qwen2.5-0.5B_dft.json b/data/dft_weights/Qwen2.5-0.5B_dft.json new file mode 100644 index 0000000000000000000000000000000000000000..e054e4760e6edc4ca35a7daff83b0641ee37dab4 --- /dev/null +++ b/data/dft_weights/Qwen2.5-0.5B_dft.json @@ -0,0 +1,126 @@ +{ + "model": "Qwen2.5-0.5B", + "theta": 1000000, + "d_head": 64, + "n_layers": 24, + "n_heads": 14, + "k_dead_boundary": 15, + "n_active_pairs": 15, + "n_dead_pairs": 17, + "band_energy": { + "E_per_pair": [ + 0.0006228263491721009, + 0.0006581460720553878, + 0.0007441935522365384, + 0.0007075510708697644, + 0.000850715034781994, + 0.0007798387478032964, + 0.0007825717954498638, + 0.0009575981093803421, + 0.0009257637793780304, + 0.0009461850683389154, + 0.0009778477457681827, + 0.0008932864851279495, + 0.0009010726880660513, + 0.0009688954760349588, + 0.0008734167919707639, + 0.0009190585327208586, + 0.0006425930632758536, + 0.000636881090940733, + 0.0007205005288900187, + 0.0007130687772587407, + 0.0007598505781061249, + 0.000745484830076748, + 0.0008069371275875407, + 0.0009048764283458391, + 0.0010603107657516375, + 0.000910289184806364, + 0.0008682869429321727, + 0.0008853495110088261, + 0.0008384072237580161, + 0.0008649438762707481, + 0.0008564445021571979, + 0.0008459685086563695 + ], + "E_dead_mean": 0.0008393272510956092, + "E_alive_mean": 0.0008223089101496347, + "snr_alive_dead": 0.9797238301228948, + "note": "dead=pairs 0..k_dead-1 (high RoPE freq, alias early); alive=k_dead..d_head/2-1" + }, + "dft_spectrum": { + "P_Q": [ + 0.05392686050433233, + 0.04275337582278279, + 0.052210872168435354, + 0.04448190225423759, + 0.05107874275816009, + 0.0421583810034171, + 0.04566938271604249, + 0.0426609299368481, + 0.04277714973748345, + 0.04147912279820549, + 0.041502405993662825, + 0.0445493971498128, + 0.040659989558539875, + 0.04626013904028544, + 0.04074890569801804, + 0.04149000007555484, + 0.04250506155788236, + 0.04281022068852725, + 0.04292522748018274, + 0.04187017661215829, + 0.041858488587725784, + 0.04173824107984558, + 0.04601647191088969, + 0.043648312991056454, + 0.05540291113956351, + 0.04145733935105237, + 0.042030692080098145, + 0.04311199132199258, + 0.042230375728376214, + 0.041016050187394286, + 0.04947889424317772, + 0.04197657080358269, + 0.05006688499227044 + ], + "P_K": [ + 0.055416975888029955, + 0.058067145220669435, + 0.06079200076452135, + 0.06030478383187954, + 0.058407092990606625, + 0.0551126759298863, + 0.058929025686780205, + 0.05345633877069333, + 0.059196941845622936, + 0.06125295303210917, + 0.05985480993293645, + 0.05639578504436934, + 0.05919310854120657, + 0.059204639219456456, + 0.05874240477755665, + 0.05750115844886938, + 0.056292761925512115, + 0.06488959075593966, + 0.05688800554709863, + 0.05571441722889422, + 0.055463547135551905, + 0.06154035954584911, + 0.06331992972280132, + 0.060086857967912834, + 0.0610530548578209, + 0.05547771625963044, + 0.05626997809646089, + 0.052312561361001854, + 0.058333648018042215, + 0.06164757807328247, + 0.056571370304961656, + 0.06322563080383563, + 0.06430757083988659 + ], + "frac_Q_alive": 0.5405299357128118, + "frac_K_alive": 0.5482035174988065, + "null_fraction": 0.5454545454545454 + }, + "elapsed_s": 5.4 +} \ No newline at end of file diff --git a/data/dft_weights/SmolLM2-360M_dft.json b/data/dft_weights/SmolLM2-360M_dft.json new file mode 100644 index 0000000000000000000000000000000000000000..a74cd330f948ce67d2cc49e01ae7db0bb717b32a --- /dev/null +++ b/data/dft_weights/SmolLM2-360M_dft.json @@ -0,0 +1,126 @@ +{ + "model": "SmolLM2-360M", + "theta": 100000, + "d_head": 64, + "n_layers": 32, + "n_heads": 15, + "k_dead_boundary": 18, + "n_active_pairs": 18, + "n_dead_pairs": 14, + "band_energy": { + "E_per_pair": [ + 0.025269863719586283, + 0.02541476932237856, + 0.027348715803236702, + 0.027804870571708308, + 0.03089532544836402, + 0.030647695797961207, + 0.032357735431287435, + 0.03409121660515666, + 0.03466076030163094, + 0.036654044431634246, + 0.037308646115707236, + 0.03618085461203009, + 0.03680996113689616, + 0.03729442257899791, + 0.037743786536157134, + 0.03635534648783505, + 0.025531852582935242, + 0.02597372094169259, + 0.02696217806369532, + 0.027827971702208742, + 0.029861762718064712, + 0.03067283452837728, + 0.032295483851339665, + 0.03348187817027792, + 0.03398339995183051, + 0.036816348327556626, + 0.03841224994393997, + 0.036484417656902225, + 0.037388850847491995, + 0.03836337691755034, + 0.037994201527908444, + 0.036750087386462835 + ], + "E_dead_mean": 0.032130199356955325, + "E_alive_mean": 0.034092502970971904, + "snr_alive_dead": 1.0610734963438913, + "note": "dead=pairs 0..k_dead-1 (high RoPE freq, alias early); alive=k_dead..d_head/2-1" + }, + "dft_spectrum": { + "P_Q": [ + 2.1210313006242094, + 2.1302040005495586, + 2.1223305023001053, + 2.1335972242410812, + 2.126314835695495, + 2.1317210010201584, + 2.128121589347068, + 2.11272735788541, + 2.1275410753358357, + 2.129873362167353, + 2.145515896188842, + 2.1190358285062025, + 2.122167575145996, + 2.111128506913698, + 2.099214426102873, + 2.1120945872217924, + 2.110947090995979, + 2.116418743390384, + 2.125797653787983, + 2.1210342543770286, + 2.1155697301062126, + 2.1087978165474466, + 2.1374832341209316, + 2.113890384523775, + 2.1251501009401825, + 2.1216704767548373, + 2.1046099548273927, + 2.1206459331771765, + 2.11173691200079, + 2.1025088942834165, + 2.1075722578215816, + 2.1178910284665995, + 2.1011656167676147 + ], + "P_K": [ + 2.3428708174199193, + 2.2987214869028287, + 2.354706438886853, + 2.2811933844210834, + 2.312777839757843, + 2.3294524004495036, + 2.3189964673719343, + 2.316389574072212, + 2.279767206034193, + 2.330349026699854, + 2.304479026280806, + 2.2813061331686266, + 2.3226374893518016, + 2.3023436155827075, + 2.3063709812668236, + 2.2981977809941916, + 2.3188293591147207, + 2.3077552767290928, + 2.2919072092909065, + 2.3268391968929576, + 2.332017850066368, + 2.299359207061272, + 2.3308832507373696, + 2.3022914116593074, + 2.2967031115186223, + 2.3071905654434626, + 2.271727303732168, + 2.307624984199611, + 2.2966894302991685, + 2.3115450369751214, + 2.256446060191771, + 2.254674117091775, + 2.311154990594025 + ], + "frac_Q_alive": 0.45378270113779084, + "frac_K_alive": 0.45328713288638145, + "null_fraction": 0.45454545454545453 + }, + "elapsed_s": 5.7 +} \ No newline at end of file diff --git a/data/dft_weights/all_results.json b/data/dft_weights/all_results.json new file mode 100644 index 0000000000000000000000000000000000000000..4aecb6dbb42b6c001e16e433bf0fc41695fb64ff --- /dev/null +++ b/data/dft_weights/all_results.json @@ -0,0 +1,416 @@ +[ + { + "model": "pythia-1b", + "theta": 10000, + "d_head": 256, + "n_layers": 16, + "n_heads": 8, + "k_dead_boundary": 90, + "n_active_pairs": 90, + "n_dead_pairs": 38, + "band_energy": { + "E_per_pair": [ + 0.0002662035290086351, + 0.0002688799099246353, + 0.00031759004264131363, + 0.00037761209546260943, + 0.00035918128207867994, + 0.00038677351824389916, + 0.00042666972694860306, + 0.00048753907117315975, + 0.0005085242449922589, + 0.0004965749164966837, + 0.0005545755848288536, + 0.0005931611761980093, + 0.0005960240380318282, + 0.0006345204928948078, + 0.0006286775500257136, + 0.0006277127301927976, + 0.0002656349985841189, + 0.0002683774811202966, + 0.0003210860713238617, + 0.0003464054821051832, + 0.0003534690650894845, + 0.0004207136572631498, + 0.0004632393226984277, + 0.0004883742133188207, + 0.0005168734796825447, + 0.0005657666342813172, + 0.000558554221470331, + 0.0006576271672429357, + 0.0006024149341783414, + 0.0006239133485905768, + 0.0006202086052553568, + 0.0006361773474736765, + 0.0006365176500366942, + 0.0006398633170192625, + 0.000636505303418744, + 0.0006276373280797998, + 0.0006291215947840101, + 0.0006391749400336266, + 0.0006320350581745515, + 0.0006364864932493219, + 0.0006340065924632654, + 0.0006380719673870772, + 0.000629806070264749, + 0.0006319283555740185, + 0.0006316360097571305, + 0.0006293077789223389, + 0.0006396738783678302, + 0.0006334340464491106, + 0.0006329104006681519, + 0.0006278635996750381, + 0.0006311798036904293, + 0.0006328442325411743, + 0.0006348571866965358, + 0.0006334225231512391, + 0.0006362194060329784, + 0.0006349930179112562, + 0.0006353335843414243, + 0.0006344274594312083, + 0.0006325475580979401, + 0.0006353618809953332, + 0.0006317681898053706, + 0.0006386199697772099, + 0.0006293945923516731, + 0.0006316489914297563, + 0.0006387587649214765, + 0.0006380424426879472, + 0.0006298458075661983, + 0.0006346435577597731, + 0.0006331320334993507, + 0.000639350555047713, + 0.0006419114926075054, + 0.0006398456629312932, + 0.0006358273949444992, + 0.0006263793330845147, + 0.0006356573642278818, + 0.0006382802121152054, + 0.0006302046256223548, + 0.000636135206605104, + 0.0006213326073520875, + 0.0006376369885856548, + 0.0006409413616665915, + 0.0006319565372905345, + 0.0006424207499549084, + 0.0006411887322883558, + 0.0006335101818422118, + 0.0006260790618171086, + 0.0006332447452450651, + 0.0006330170441515293, + 0.0006340915614373444, + 0.0006389569446128007, + 0.0006376424473728548, + 0.0006332403313535906, + 0.0006369829081904754, + 0.0006410580604097049, + 0.0006361083010233415, + 0.0006389413447323022, + 0.0006371255724388902, + 0.0006378392172337044, + 0.0006386277519823125, + 0.0006393495067413824, + 0.0006323519314719306, + 0.0006370026744662027, + 0.0006338470095670345, + 0.0006304903785121496, + 0.0006374731979121862, + 0.0006313238932307286, + 0.0006372643139229694, + 0.0006364899797972612, + 0.0006364324498235874, + 0.0006384530478271699, + 0.0006345538554342056, + 0.000631931921816431, + 0.0006298544249148108, + 0.0006388854621945939, + 0.0006236617937247502, + 0.0006359592782700929, + 0.0006353463745654153, + 0.0006320260915799736, + 0.0006383923484918341, + 0.0006359760359373468, + 0.000631259185752242, + 0.0006381875986107843, + 0.0006352826064812689, + 0.0006337700044696248, + 0.0006376574293653903, + 0.0006331568342829996, + 0.000631784545021219, + 0.0006354958067049665 + ], + "E_dead_mean": 0.0005781116187696019, + "E_alive_mean": 0.0006352954188323087, + "snr_alive_dead": 1.098914806599972, + "note": "dead=pairs 0..k_dead-1 (high RoPE freq, alias early); alive=k_dead..d_head/2-1" + }, + "dft_spectrum": { + "P_Q": [ + 0.14531712601163851, + 0.14559681446067646, + 0.14646645694368995, + 0.1463145024299945, + 0.14558252897574808, + 0.14390172418606628, + 0.1451889822716827, + 0.1454678156623636, + 0.14551633695042318, + 0.14631252126566852, + 0.14658481923312122, + 0.14726373575420318, + 0.1456463214708481, + 0.14541322070667878, + 0.14649395578611746, + 0.14719421885160536, + 0.14525229640532533, + 0.14616442921207728, + 0.1479197004002997, + 0.1466003190135848, + 0.14560799726746074, + 0.144418500561758, + 0.14667330700850698, + 0.14589698528543096, + 0.1449982627976646, + 0.1459746870686337, + 0.14570077179764057, + 0.14645300863164545, + 0.14472969171273584, + 0.14615296276667641, + 0.14686599441449194, + 0.14651179553199822, + 0.1480220950491709, + 0.1474227337396873, + 0.145174031910865, + 0.14803371751552583, + 0.14676910112763952, + 0.1465458990585078, + 0.1470198832829111, + 0.14827683035790457, + 0.14735687945047998, + 0.14808196226520057, + 0.14657230791310588, + 0.1447272988433252, + 0.14711335381638657, + 0.1453991384930435, + 0.14526475745965814, + 0.14586859824118087, + 0.14758384997988722, + 0.1469716733989527, + 0.1463820388777532, + 0.14688214625154294, + 0.14652883183284, + 0.14570150477834587, + 0.1473172666258702, + 0.14737273412489968, + 0.14716470108131158, + 0.14693905924006112, + 0.14564652680333082, + 0.14600142768552365, + 0.14610510320288306, + 0.1458434697043151, + 0.14622312929919917, + 0.14877889278180634, + 0.14574368709167262, + 0.14614083075410222, + 0.14517480920039316, + 0.1449507005996301, + 0.14663968506352404, + 0.14722164662837373, + 0.14651014290015557, + 0.1471402744068547, + 0.14780423883151236, + 0.14663525687240425, + 0.14498043263406687, + 0.1461524763321965, + 0.14517319343627327, + 0.14597074916626906, + 0.14663387344964418, + 0.1466988461237172, + 0.14642955058161136, + 0.1470993489313116, + 0.1449936740830339, + 0.14643904781639738, + 0.14559035187934155, + 0.14429079604565725, + 0.14536637486856882, + 0.14624272506077046, + 0.14471705251939437, + 0.14567474959677848, + 0.14552911279096586, + 0.14813967768900754, + 0.14472857553986057, + 0.14704985342428195, + 0.14596846098797164, + 0.14535816648410357, + 0.14588705492701182, + 0.14665471355732895, + 0.14739649485940004, + 0.1464505472060551, + 0.14695123668237953, + 0.1466458692863667, + 0.14561921823499122, + 0.14816174498447288, + 0.14651908371198796, + 0.14682149323934626, + 0.1475995552099748, + 0.1451801467220718, + 0.14522535181591886, + 0.14533411266047272, + 0.14530705500853286, + 0.14674347272473698, + 0.14577305520199707, + 0.14806786349849727, + 0.14528683593099095, + 0.1451423849483936, + 0.14659941336220617, + 0.14577107189644187, + 0.14610175631218708, + 0.1459674759175672, + 0.14810195426699016, + 0.14655503924884186, + 0.1458024620290366, + 0.14665913884423543, + 0.1474209937660474, + 0.1446865754547964, + 0.1463033495959687, + 0.14691965928934247, + 0.14476954097850503 + ], + "P_K": [ + 0.14939435297637665, + 0.14758600725182233, + 0.14639177324006347, + 0.14760337666892806, + 0.14895810161719664, + 0.1476855507525895, + 0.14869638606215874, + 0.1480180603127046, + 0.148008900668536, + 0.1475190639710286, + 0.14760260557211582, + 0.14789550809914973, + 0.14689051455598517, + 0.14863620627785773, + 0.14906720570639045, + 0.14831266321653008, + 0.14781969091282335, + 0.14770266848680288, + 0.1483638603194859, + 0.15000251641658885, + 0.14833935614007837, + 0.14961302407659685, + 0.14742042252646761, + 0.14855750145211533, + 0.149567977079548, + 0.1499942316451513, + 0.14862602719891033, + 0.14733765921014358, + 0.14776529722314694, + 0.14803087668589007, + 0.14876406199949868, + 0.14798948898877157, + 0.1471898544539652, + 0.1492717766758136, + 0.15028055507347624, + 0.14714762542283247, + 0.14861928925746254, + 0.1476990874381594, + 0.14768729225006455, + 0.1455062973711232, + 0.14777349185257826, + 0.14822116822644277, + 0.14748604136487628, + 0.14861507666461773, + 0.14897227415217476, + 0.14728181422305264, + 0.1467695438884458, + 0.1471034376741243, + 0.14767183432046657, + 0.14912448016813645, + 0.1474844144997965, + 0.14766051852665027, + 0.1480043379128055, + 0.14886288078759283, + 0.14728413323702938, + 0.1465102450555884, + 0.14809220829706793, + 0.1494426745374135, + 0.14869134327122496, + 0.14848802761692978, + 0.1484356551637274, + 0.1474196522894141, + 0.14890649733633057, + 0.14698891170536255, + 0.14800317729670748, + 0.14679755595490834, + 0.1470315303403847, + 0.1465194978877543, + 0.1470659008803511, + 0.1461386519308877, + 0.14661628076034383, + 0.14715627340103496, + 0.14808726058342192, + 0.14749716892463993, + 0.1464238965758078, + 0.147665951671268, + 0.14921720026854893, + 0.14839762138471357, + 0.14834595761141833, + 0.14751155440654004, + 0.14867695496589395, + 0.14895390814826462, + 0.14713778576182565, + 0.1485369839516481, + 0.1484377558579365, + 0.14774346754244533, + 0.14890231525015157, + 0.1492957335920888, + 0.14858643654455383, + 0.14859858928217323, + 0.14828458782276405, + 0.1478033190019271, + 0.14719671404592133, + 0.14813818956912836, + 0.14864336301161893, + 0.14847478392798596, + 0.14909793619323303, + 0.14757219454067036, + 0.14749491712353355, + 0.14908245391824107, + 0.15005995892833993, + 0.14890722613685511, + 0.148733019434291, + 0.1482268354254344, + 0.14868330850591907, + 0.14872979267789413, + 0.14883849809409552, + 0.1477564605831683, + 0.1479973992938898, + 0.14763258466456075, + 0.14896137440179474, + 0.14918448117273453, + 0.14780015787385659, + 0.14699871128883485, + 0.1466214695391534, + 0.1482415504111948, + 0.14874597292108932, + 0.14786502935017032, + 0.14767567954203029, + 0.1481039150242517, + 0.14821490561837694, + 0.1502824662746978, + 0.1490364093031466, + 0.1484160851140081, + 0.14875398265911882, + 0.14934862635418267, + 0.1478971181885654, + 0.14873223277246433, + 0.1473340098900525 + ], + "frac_Q_alive": 0.3023922082782471, + "frac_K_alive": 0.3027861847564636, + "null_fraction": 0.3023255813953488 + }, + "elapsed_s": 5.9 + } +] \ No newline at end of file diff --git a/data/dft_weights/pythia-160m_dft.json b/data/dft_weights/pythia-160m_dft.json new file mode 100644 index 0000000000000000000000000000000000000000..92d7a5b8d976afac5bd9790b9e39e186e2beb6e1 --- /dev/null +++ b/data/dft_weights/pythia-160m_dft.json @@ -0,0 +1,126 @@ +{ + "model": "pythia-160m", + "theta": 10000, + "d_head": 64, + "n_layers": 12, + "n_heads": 12, + "k_dead_boundary": 23, + "n_active_pairs": 23, + "n_dead_pairs": 9, + "band_energy": { + "E_per_pair": [ + 0.001064130950706183, + 0.0012110301795473788, + 0.0014869011689976712, + 0.0020551237402186315, + 0.0008975173361452309, + 0.0012307940141909057, + 0.0013022122246992593, + 0.001751014025002304, + 0.010412280780454038, + 0.009159020898045532, + 0.009579110671135519, + 0.012873442889233248, + 0.007848018351069186, + 0.00977274171398474, + 0.008646933923753548, + 0.010683184749521187, + 0.00916713065008581, + 0.007902943464513455, + 0.011261846996300543, + 0.0080462609451691, + 0.010197897312739061, + 0.008272148262929276, + 0.008684634919417376, + 0.009037639047568923, + 0.008514541465653261, + 0.010570794737254295, + 0.00927946967729238, + 0.010585213738320616, + 0.008661203316781515, + 0.009069388265844382, + 0.009947797783145992, + 0.010626469744036311 + ], + "E_dead_mean": 0.006674187833385181, + "E_alive_mean": 0.009588057530655296, + "snr_alive_dead": 1.4365879068098684, + "note": "dead=pairs 0..k_dead-1 (high RoPE freq, alias early); alive=k_dead..d_head/2-1" + }, + "dft_spectrum": { + "P_Q": [ + 0.2958437570347699, + 0.348343640592764, + 0.24555797281491, + 0.40297945607279995, + 0.322743520206249, + 0.2869425880994953, + 0.37247994033492415, + 0.2954177619982386, + 0.38897240244554204, + 0.3294966998674521, + 0.38441006827008345, + 0.32604674450177895, + 0.25162863779837724, + 0.31672071094961624, + 0.3259164751708582, + 0.36325387051652336, + 0.31398786570793763, + 0.3476528222751796, + 0.31054765274936663, + 0.291829167010772, + 0.3515384377239891, + 0.361517186066457, + 0.3553785084870785, + 0.3291896297574752, + 0.2950313963547555, + 0.2832946113010028, + 0.35011134576448194, + 0.30321948033327806, + 0.2827686790744266, + 0.3231308427027485, + 0.2991999331418189, + 0.33306226281733936, + 0.3181819969321611 + ], + "P_K": [ + 0.2895982372456034, + 0.2557708329645376, + 0.2779893377262116, + 0.25784963421521795, + 0.258599459903312, + 0.25807211945560005, + 0.28222506528559765, + 0.2388104718094088, + 0.3279792482943225, + 0.2881836546519818, + 0.2623019846268205, + 0.2527246975959595, + 0.2455083639035442, + 0.2907001318114804, + 0.25672807900454153, + 0.24306433128687477, + 0.28588553283187657, + 0.25444223470832467, + 0.2497074819244824, + 0.2723151532282038, + 0.292947676607296, + 0.3387375096034239, + 0.24785415281447978, + 0.25833108988957815, + 0.22527443779193257, + 0.24133667301794404, + 0.2777118990890367, + 0.29787607396914834, + 0.31693569559542367, + 0.2808322269062053, + 0.22895407033230444, + 0.3319835111449873, + 0.32165428335043195 + ], + "frac_Q_alive": 0.29115214487590435, + "frac_K_alive": 0.3086830226214835, + "null_fraction": 0.30303030303030304 + }, + "elapsed_s": 4.9 +} \ No newline at end of file diff --git a/data/dft_weights/pythia-1b_dft.json b/data/dft_weights/pythia-1b_dft.json new file mode 100644 index 0000000000000000000000000000000000000000..036d923c18b0ed77e08f963a8cf73b88c6e35566 --- /dev/null +++ b/data/dft_weights/pythia-1b_dft.json @@ -0,0 +1,414 @@ +{ + "model": "pythia-1b", + "theta": 10000, + "d_head": 256, + "n_layers": 16, + "n_heads": 8, + "k_dead_boundary": 90, + "n_active_pairs": 90, + "n_dead_pairs": 38, + "band_energy": { + "E_per_pair": [ + 0.0002662035290086351, + 0.0002688799099246353, + 0.00031759004264131363, + 0.00037761209546260943, + 0.00035918128207867994, + 0.00038677351824389916, + 0.00042666972694860306, + 0.00048753907117315975, + 0.0005085242449922589, + 0.0004965749164966837, + 0.0005545755848288536, + 0.0005931611761980093, + 0.0005960240380318282, + 0.0006345204928948078, + 0.0006286775500257136, + 0.0006277127301927976, + 0.0002656349985841189, + 0.0002683774811202966, + 0.0003210860713238617, + 0.0003464054821051832, + 0.0003534690650894845, + 0.0004207136572631498, + 0.0004632393226984277, + 0.0004883742133188207, + 0.0005168734796825447, + 0.0005657666342813172, + 0.000558554221470331, + 0.0006576271672429357, + 0.0006024149341783414, + 0.0006239133485905768, + 0.0006202086052553568, + 0.0006361773474736765, + 0.0006365176500366942, + 0.0006398633170192625, + 0.000636505303418744, + 0.0006276373280797998, + 0.0006291215947840101, + 0.0006391749400336266, + 0.0006320350581745515, + 0.0006364864932493219, + 0.0006340065924632654, + 0.0006380719673870772, + 0.000629806070264749, + 0.0006319283555740185, + 0.0006316360097571305, + 0.0006293077789223389, + 0.0006396738783678302, + 0.0006334340464491106, + 0.0006329104006681519, + 0.0006278635996750381, + 0.0006311798036904293, + 0.0006328442325411743, + 0.0006348571866965358, + 0.0006334225231512391, + 0.0006362194060329784, + 0.0006349930179112562, + 0.0006353335843414243, + 0.0006344274594312083, + 0.0006325475580979401, + 0.0006353618809953332, + 0.0006317681898053706, + 0.0006386199697772099, + 0.0006293945923516731, + 0.0006316489914297563, + 0.0006387587649214765, + 0.0006380424426879472, + 0.0006298458075661983, + 0.0006346435577597731, + 0.0006331320334993507, + 0.000639350555047713, + 0.0006419114926075054, + 0.0006398456629312932, + 0.0006358273949444992, + 0.0006263793330845147, + 0.0006356573642278818, + 0.0006382802121152054, + 0.0006302046256223548, + 0.000636135206605104, + 0.0006213326073520875, + 0.0006376369885856548, + 0.0006409413616665915, + 0.0006319565372905345, + 0.0006424207499549084, + 0.0006411887322883558, + 0.0006335101818422118, + 0.0006260790618171086, + 0.0006332447452450651, + 0.0006330170441515293, + 0.0006340915614373444, + 0.0006389569446128007, + 0.0006376424473728548, + 0.0006332403313535906, + 0.0006369829081904754, + 0.0006410580604097049, + 0.0006361083010233415, + 0.0006389413447323022, + 0.0006371255724388902, + 0.0006378392172337044, + 0.0006386277519823125, + 0.0006393495067413824, + 0.0006323519314719306, + 0.0006370026744662027, + 0.0006338470095670345, + 0.0006304903785121496, + 0.0006374731979121862, + 0.0006313238932307286, + 0.0006372643139229694, + 0.0006364899797972612, + 0.0006364324498235874, + 0.0006384530478271699, + 0.0006345538554342056, + 0.000631931921816431, + 0.0006298544249148108, + 0.0006388854621945939, + 0.0006236617937247502, + 0.0006359592782700929, + 0.0006353463745654153, + 0.0006320260915799736, + 0.0006383923484918341, + 0.0006359760359373468, + 0.000631259185752242, + 0.0006381875986107843, + 0.0006352826064812689, + 0.0006337700044696248, + 0.0006376574293653903, + 0.0006331568342829996, + 0.000631784545021219, + 0.0006354958067049665 + ], + "E_dead_mean": 0.0005781116187696019, + "E_alive_mean": 0.0006352954188323087, + "snr_alive_dead": 1.098914806599972, + "note": "dead=pairs 0..k_dead-1 (high RoPE freq, alias early); alive=k_dead..d_head/2-1" + }, + "dft_spectrum": { + "P_Q": [ + 0.14531712601163851, + 0.14559681446067646, + 0.14646645694368995, + 0.1463145024299945, + 0.14558252897574808, + 0.14390172418606628, + 0.1451889822716827, + 0.1454678156623636, + 0.14551633695042318, + 0.14631252126566852, + 0.14658481923312122, + 0.14726373575420318, + 0.1456463214708481, + 0.14541322070667878, + 0.14649395578611746, + 0.14719421885160536, + 0.14525229640532533, + 0.14616442921207728, + 0.1479197004002997, + 0.1466003190135848, + 0.14560799726746074, + 0.144418500561758, + 0.14667330700850698, + 0.14589698528543096, + 0.1449982627976646, + 0.1459746870686337, + 0.14570077179764057, + 0.14645300863164545, + 0.14472969171273584, + 0.14615296276667641, + 0.14686599441449194, + 0.14651179553199822, + 0.1480220950491709, + 0.1474227337396873, + 0.145174031910865, + 0.14803371751552583, + 0.14676910112763952, + 0.1465458990585078, + 0.1470198832829111, + 0.14827683035790457, + 0.14735687945047998, + 0.14808196226520057, + 0.14657230791310588, + 0.1447272988433252, + 0.14711335381638657, + 0.1453991384930435, + 0.14526475745965814, + 0.14586859824118087, + 0.14758384997988722, + 0.1469716733989527, + 0.1463820388777532, + 0.14688214625154294, + 0.14652883183284, + 0.14570150477834587, + 0.1473172666258702, + 0.14737273412489968, + 0.14716470108131158, + 0.14693905924006112, + 0.14564652680333082, + 0.14600142768552365, + 0.14610510320288306, + 0.1458434697043151, + 0.14622312929919917, + 0.14877889278180634, + 0.14574368709167262, + 0.14614083075410222, + 0.14517480920039316, + 0.1449507005996301, + 0.14663968506352404, + 0.14722164662837373, + 0.14651014290015557, + 0.1471402744068547, + 0.14780423883151236, + 0.14663525687240425, + 0.14498043263406687, + 0.1461524763321965, + 0.14517319343627327, + 0.14597074916626906, + 0.14663387344964418, + 0.1466988461237172, + 0.14642955058161136, + 0.1470993489313116, + 0.1449936740830339, + 0.14643904781639738, + 0.14559035187934155, + 0.14429079604565725, + 0.14536637486856882, + 0.14624272506077046, + 0.14471705251939437, + 0.14567474959677848, + 0.14552911279096586, + 0.14813967768900754, + 0.14472857553986057, + 0.14704985342428195, + 0.14596846098797164, + 0.14535816648410357, + 0.14588705492701182, + 0.14665471355732895, + 0.14739649485940004, + 0.1464505472060551, + 0.14695123668237953, + 0.1466458692863667, + 0.14561921823499122, + 0.14816174498447288, + 0.14651908371198796, + 0.14682149323934626, + 0.1475995552099748, + 0.1451801467220718, + 0.14522535181591886, + 0.14533411266047272, + 0.14530705500853286, + 0.14674347272473698, + 0.14577305520199707, + 0.14806786349849727, + 0.14528683593099095, + 0.1451423849483936, + 0.14659941336220617, + 0.14577107189644187, + 0.14610175631218708, + 0.1459674759175672, + 0.14810195426699016, + 0.14655503924884186, + 0.1458024620290366, + 0.14665913884423543, + 0.1474209937660474, + 0.1446865754547964, + 0.1463033495959687, + 0.14691965928934247, + 0.14476954097850503 + ], + "P_K": [ + 0.14939435297637665, + 0.14758600725182233, + 0.14639177324006347, + 0.14760337666892806, + 0.14895810161719664, + 0.1476855507525895, + 0.14869638606215874, + 0.1480180603127046, + 0.148008900668536, + 0.1475190639710286, + 0.14760260557211582, + 0.14789550809914973, + 0.14689051455598517, + 0.14863620627785773, + 0.14906720570639045, + 0.14831266321653008, + 0.14781969091282335, + 0.14770266848680288, + 0.1483638603194859, + 0.15000251641658885, + 0.14833935614007837, + 0.14961302407659685, + 0.14742042252646761, + 0.14855750145211533, + 0.149567977079548, + 0.1499942316451513, + 0.14862602719891033, + 0.14733765921014358, + 0.14776529722314694, + 0.14803087668589007, + 0.14876406199949868, + 0.14798948898877157, + 0.1471898544539652, + 0.1492717766758136, + 0.15028055507347624, + 0.14714762542283247, + 0.14861928925746254, + 0.1476990874381594, + 0.14768729225006455, + 0.1455062973711232, + 0.14777349185257826, + 0.14822116822644277, + 0.14748604136487628, + 0.14861507666461773, + 0.14897227415217476, + 0.14728181422305264, + 0.1467695438884458, + 0.1471034376741243, + 0.14767183432046657, + 0.14912448016813645, + 0.1474844144997965, + 0.14766051852665027, + 0.1480043379128055, + 0.14886288078759283, + 0.14728413323702938, + 0.1465102450555884, + 0.14809220829706793, + 0.1494426745374135, + 0.14869134327122496, + 0.14848802761692978, + 0.1484356551637274, + 0.1474196522894141, + 0.14890649733633057, + 0.14698891170536255, + 0.14800317729670748, + 0.14679755595490834, + 0.1470315303403847, + 0.1465194978877543, + 0.1470659008803511, + 0.1461386519308877, + 0.14661628076034383, + 0.14715627340103496, + 0.14808726058342192, + 0.14749716892463993, + 0.1464238965758078, + 0.147665951671268, + 0.14921720026854893, + 0.14839762138471357, + 0.14834595761141833, + 0.14751155440654004, + 0.14867695496589395, + 0.14895390814826462, + 0.14713778576182565, + 0.1485369839516481, + 0.1484377558579365, + 0.14774346754244533, + 0.14890231525015157, + 0.1492957335920888, + 0.14858643654455383, + 0.14859858928217323, + 0.14828458782276405, + 0.1478033190019271, + 0.14719671404592133, + 0.14813818956912836, + 0.14864336301161893, + 0.14847478392798596, + 0.14909793619323303, + 0.14757219454067036, + 0.14749491712353355, + 0.14908245391824107, + 0.15005995892833993, + 0.14890722613685511, + 0.148733019434291, + 0.1482268354254344, + 0.14868330850591907, + 0.14872979267789413, + 0.14883849809409552, + 0.1477564605831683, + 0.1479973992938898, + 0.14763258466456075, + 0.14896137440179474, + 0.14918448117273453, + 0.14780015787385659, + 0.14699871128883485, + 0.1466214695391534, + 0.1482415504111948, + 0.14874597292108932, + 0.14786502935017032, + 0.14767567954203029, + 0.1481039150242517, + 0.14821490561837694, + 0.1502824662746978, + 0.1490364093031466, + 0.1484160851140081, + 0.14875398265911882, + 0.14934862635418267, + 0.1478971181885654, + 0.14873223277246433, + 0.1473340098900525 + ], + "frac_Q_alive": 0.3023922082782471, + "frac_K_alive": 0.3027861847564636, + "null_fraction": 0.3023255813953488 + }, + "elapsed_s": 5.9 +} \ No newline at end of file diff --git a/data/dft_weights/pythia-70m_dft.json b/data/dft_weights/pythia-70m_dft.json new file mode 100644 index 0000000000000000000000000000000000000000..be8474d87565fd7ebbb041fec5d7e6cfb43ce3af --- /dev/null +++ b/data/dft_weights/pythia-70m_dft.json @@ -0,0 +1,126 @@ +{ + "model": "pythia-70m", + "theta": 10000, + "d_head": 64, + "n_layers": 6, + "n_heads": 8, + "k_dead_boundary": 23, + "n_active_pairs": 23, + "n_dead_pairs": 9, + "band_energy": { + "E_per_pair": [ + 0.0013869976176768735, + 0.0015220016487849837, + 0.002320462079903033, + 0.002168280272599077, + 0.0011673600702124531, + 0.0017694515881885309, + 0.0015121884977512916, + 0.0029346827504923567, + 0.022924842181964777, + 0.022199681117247867, + 0.029184473507484654, + 0.029738308420443598, + 0.024061748534829046, + 0.03451017173453389, + 0.028540510100962518, + 0.02241011993100983, + 0.03376202155171389, + 0.02385499979330537, + 0.025953570800387144, + 0.027181807020194054, + 0.027968790357893642, + 0.019702527294915246, + 0.041393361272639595, + 0.023811495838648018, + 0.024168842468498042, + 0.02488592621254308, + 0.027008082821945816, + 0.028346531313824624, + 0.03716946941494825, + 0.03340487747360991, + 0.02535810896855158, + 0.02637646782871646 + ], + "E_dead_mean": 0.01861601557152755, + "E_alive_mean": 0.027836644704587306, + "snr_alive_dead": 1.4953062644439894, + "note": "dead=pairs 0..k_dead-1 (high RoPE freq, alias early); alive=k_dead..d_head/2-1" + }, + "dft_spectrum": { + "P_Q": [ + 1.6717104003371397, + 1.105004299059329, + 0.7024620839092242, + 0.6807966029214289, + 0.6914032773598832, + 1.0257135972664948, + 0.8916676485653662, + 0.9339581702965415, + 1.137243269635677, + 1.5228601771867407, + 0.8253172526210467, + 1.2022379840547384, + 1.3831510325227498, + 0.675702239833746, + 1.2202923201242881, + 0.6528626150036131, + 0.670455416055383, + 0.7497026861547055, + 0.9280180480991999, + 1.0392323199313223, + 0.4483880778183274, + 0.46287084148244667, + 0.8009219229122845, + 0.3452236851925979, + 0.6441819505206388, + 0.838249741144948, + 0.6740314620596012, + 0.8825840307188834, + 0.7261948403621759, + 0.5157278166540087, + 0.45555459601531306, + 0.6803138713776935, + 0.9512198956774439 + ], + "P_K": [ + 1.3130901707615914, + 1.3606315944013698, + 1.3644039695222967, + 0.7183332606141487, + 1.2256231825812745, + 0.7279920709298087, + 0.8380431061306055, + 0.8684510764735901, + 0.9971720116889409, + 1.153523899650725, + 0.8642842450302873, + 0.8198888227690156, + 1.3093188303906966, + 1.2564435630627921, + 0.7908796056793129, + 1.8742300239723149, + 1.077268796248083, + 0.9092157649442464, + 1.0146687915722363, + 1.0769092585498339, + 1.7463530656465398, + 0.5475784764325449, + 0.6924836583751639, + 0.9134259243453201, + 1.4194316969844074, + 0.9655504549865862, + 2.6350416556014165, + 1.1491870123181223, + 0.7587285922116842, + 0.8719527498413142, + 1.2022106321429469, + 1.5856937781773486, + 1.2527629137097296 + ], + "frac_Q_alive": 0.2386074726204286, + "frac_K_alive": 0.3419228209567327, + "null_fraction": 0.30303030303030304 + }, + "elapsed_s": 4.5 +} \ No newline at end of file diff --git a/data/dict1_primitives/gpt2-medium_dict1.json b/data/dict1_primitives/gpt2-medium_dict1.json new file mode 100644 index 0000000000000000000000000000000000000000..b006374af32db9645a50916cc43fd777d3c5201c --- /dev/null +++ b/data/dict1_primitives/gpt2-medium_dict1.json @@ -0,0 +1,6 @@ +{ + "model": "gpt2-medium", + "L_crit": 23, + "N_heads": 16, + "D_model": 1024, + "control_mean_cosim": \ No newline at end of file diff --git a/data/dict1_primitives/gpt2-medium_dict2.json b/data/dict1_primitives/gpt2-medium_dict2.json new file mode 100644 index 0000000000000000000000000000000000000000..4546f0fb15c82d4d2b1b45416c055f218ab6b769 --- /dev/null +++ b/data/dict1_primitives/gpt2-medium_dict2.json @@ -0,0 +1,380 @@ +{ + "model": "gpt2-medium", + "L_crit": 23, + "L_low": 7, + "ctrl_ceiling": 0.06966559588909149, + "n_unique_combos": 7, + "n_signal": 6, + "mean_snr": 2.969398949906947, + "conjecture_supported": true, + "primitives": { + "NEGATION": { + "best_layer": 8, + "best_head": 7, + "cos_best": 0.13329872488975525, + "cos_2nd": 0.09282198548316956, + "specificity": 1.4360684352938828, + "signal_snr": 1.9134082078126524, + "cos_profile": { + "7": 0.04525420069694519, + "8": 0.13329872488975525, + "9": 0.05483391135931015, + "10": 0.0696668028831482, + "11": 0.0945112556219101, + "12": 0.1170174777507782, + "13": 0.13116368651390076, + "14": 0.1139974370598793, + "15": 0.10650475323200226, + "16": 0.09897567331790924, + "17": 0.09799027442932129, + "18": 0.08689571171998978, + "19": 0.06172046437859535, + "20": 0.11230587959289551, + "21": 0.11145536601543427, + "22": 0.08273802697658539, + "23": 0.07340490818023682 + }, + "cos_per_head_at_best_layer": [ + 0.00856618769466877, + 0.056749194860458374, + 0.046669092029333115, + 0.008669473230838776, + 0.004219304770231247, + 0.030297869816422462, + 0.09282198548316956, + 0.13329872488975525, + 0.027212930843234062, + 0.01690634898841381, + 0.05086136609315872, + 0.009447619318962097, + 0.048460546880960464, + 0.0022882698103785515, + 0.01668638363480568, + 0.02604120783507824 + ], + "n_pairs": 20 + }, + "CAUSATION": { + "best_layer": 16, + "best_head": 15, + "cos_best": 0.15980321168899536, + "cos_2nd": 0.09578980505466461, + "specificity": 1.6682694982992246, + "signal_snr": 2.29386122885596, + "cos_profile": { + "7": 0.0561734139919281, + "8": 0.036950815469026566, + "9": 0.042603716254234314, + "10": 0.11084011197090149, + "11": 0.0918252170085907, + "12": 0.07896828651428223, + "13": 0.09560944139957428, + "14": 0.13985705375671387, + "15": 0.11361199617385864, + "16": 0.15980321168899536, + "17": 0.11937230825424194, + "18": 0.11294577270746231, + "19": 0.10807356983423233, + "20": 0.14007869362831116, + "21": 0.11365629732608795, + "22": 0.10150214284658432, + "23": 0.06394678354263306 + }, + "cos_per_head_at_best_layer": [ + 0.020369071513414383, + 0.00751408189535141, + 0.09578980505466461, + 0.007023860700428486, + 0.0021175071597099304, + 0.09446202218532562, + 0.01924462988972664, + 0.05881129205226898, + 0.018624689429998398, + 0.030989238992333412, + 0.06243111193180084, + 0.03067045658826828, + 0.0465470589697361, + 0.005830363370478153, + 0.061823565512895584, + 0.15980321168899536 + ], + "n_pairs": 31 + }, + "TEMPORAL_BEFORE": { + "best_layer": 20, + "best_head": 6, + "cos_best": 0.15317009389400482, + "cos_2nd": 0.10289546847343445, + "specificity": 1.4885990090511252, + "signal_snr": 2.198647549634198, + "cos_profile": { + "7": 0.06136360764503479, + "8": 0.06350380182266235, + "9": 0.08382818102836609, + "10": 0.11491823941469193, + "11": 0.0781867578625679, + "12": 0.11954285204410553, + "13": 0.09792529791593552, + "14": 0.09539800137281418, + "15": 0.12698349356651306, + "16": 0.14621295034885406, + "17": 0.12276062369346619, + "18": 0.09075461328029633, + "19": 0.12810364365577698, + "20": 0.15317009389400482, + "21": 0.10625740885734558, + "22": 0.08459354937076569, + "23": 0.08282521367073059 + }, + "cos_per_head_at_best_layer": [ + 0.014623652212321758, + 0.0032026823610067368, + 0.004678389523178339, + 0.023717302829027176, + 0.024002229794859886, + 0.03287215903401375, + 0.15317009389400482, + 0.024989157915115356, + 0.006702563259750605, + 0.017013926059007645, + 0.08834727108478546, + 0.060744088143110275, + 0.10289546847343445, + 0.06877174228429794, + 0.0010158875957131386, + 0.09605099260807037 + ], + "n_pairs": 27 + }, + "CONDITION": { + "best_layer": 19, + "best_head": 14, + "cos_best": 0.09429794549942017, + "cos_2nd": 0.05547429621219635, + "specificity": 1.6998493038806486, + "signal_snr": 1.3535798114174444, + "cos_profile": { + "7": 0.04086914286017418, + "8": 0.08249269425868988, + "9": 0.06867826730012894, + "10": 0.0706104189157486, + "11": 0.032927826046943665, + "12": 0.07956945896148682, + "13": 0.06336650997400284, + "14": 0.0697469413280487, + "15": 0.07865284383296967, + "16": 0.04585403576493263, + "17": 0.058212168514728546, + "18": 0.04280932620167732, + "19": 0.09429794549942017, + "20": 0.07190367579460144, + "21": 0.05567653477191925, + "22": 0.05346561223268509, + "23": 0.05783019959926605 + }, + "cos_per_head_at_best_layer": [ + 0.02300499752163887, + 0.05280838906764984, + 0.028871970251202583, + 0.024449847638607025, + 0.03016638197004795, + 0.03136441111564636, + 0.024813063442707062, + 0.04542487859725952, + 0.04513610154390335, + 0.03542342782020569, + 0.05547429621219635, + 0.043486665934324265, + 0.033980000764131546, + 0.03759770095348358, + 0.09429794549942017, + 0.033536262810230255 + ], + "n_pairs": 36 + }, + "AGENT_PATIENT": { + "best_layer": 16, + "best_head": 15, + "cos_best": 0.39677897095680237, + "cos_2nd": 0.1517615020275116, + "specificity": 2.6144902563654338, + "signal_snr": 5.695479385448738, + "cos_profile": { + "7": 0.13660548627376556, + "8": 0.3155892789363861, + "9": 0.09103474766016006, + "10": 0.2530129551887512, + "11": 0.3312154710292816, + "12": 0.3958240747451782, + "13": 0.33755379915237427, + "14": 0.33262190222740173, + "15": 0.39400678873062134, + "16": 0.39677897095680237, + "17": 0.20999521017074585, + "18": 0.17032361030578613, + "19": 0.1440802812576294, + "20": 0.2523527145385742, + "21": 0.17079833149909973, + "22": 0.19242817163467407, + "23": 0.07614666223526001 + }, + "cos_per_head_at_best_layer": [ + 0.027398794889450073, + 0.07386967539787292, + 0.09205180406570435, + 0.0012716148048639297, + 0.0347403846681118, + 0.05294525623321533, + 0.01750430092215538, + 0.05344638228416443, + 0.05930948257446289, + 0.012693461030721664, + 0.13516774773597717, + 0.012687666341662407, + 0.10483114421367645, + 0.08136922121047974, + 0.1517615020275116, + 0.39677897095680237 + ], + "n_pairs": 40 + }, + "QUANTITY_SCOPE": { + "best_layer": 15, + "best_head": 14, + "cos_best": 0.47169432044029236, + "cos_2nd": 0.11587178707122803, + "specificity": 4.070829735969339, + "signal_snr": 6.770835842994867, + "cos_profile": { + "7": 0.18884512782096863, + "8": 0.3904935121536255, + "9": 0.1288873255252838, + "10": 0.29310670495033264, + "11": 0.3706527650356293, + "12": 0.43829473853111267, + "13": 0.4382786154747009, + "14": 0.4325908124446869, + "15": 0.47169432044029236, + "16": 0.4460446238517761, + "17": 0.25641927123069763, + "18": 0.18440043926239014, + "19": 0.17968517541885376, + "20": 0.2318682223558426, + "21": 0.1279309242963791, + "22": 0.14787901937961578, + "23": 0.06604835391044617 + }, + "cos_per_head_at_best_layer": [ + 0.037250954657793045, + 0.05001065135002136, + 0.06663894653320312, + 0.06605330109596252, + 0.11019263416528702, + 0.09962573647499084, + 0.05508285015821457, + 0.035468362271785736, + 0.09557762742042542, + 0.04356438294053078, + 0.11587178707122803, + 0.07746200263500214, + 0.025552429258823395, + 0.028516827151179314, + 0.47169432044029236, + 0.03325724974274635 + ], + "n_pairs": 20 + }, + "EPISTEMIC_CERTAINTY": { + "best_layer": 8, + "best_head": 15, + "cos_best": 0.10431190580129623, + "cos_2nd": 0.07404517382383347, + "specificity": 1.4087603419057713, + "signal_snr": 1.4973230756547182, + "cos_profile": { + "7": 0.04732213914394379, + "8": 0.10431190580129623, + "9": 0.05332563444972038, + "10": 0.05548856779932976, + "11": 0.0794324278831482, + "12": 0.06899464875459671, + "13": 0.06735017150640488, + "14": 0.08548823744058609, + "15": 0.08098684251308441, + "16": 0.060586124658584595, + "17": 0.10010843724012375, + "18": 0.0603463351726532, + "19": 0.08847423642873764, + "20": 0.102689228951931, + "21": 0.09566798806190491, + "22": 0.05737375095486641, + "23": 0.08662758022546768 + }, + "cos_per_head_at_best_layer": [ + 0.03846576064825058, + 0.05552490055561066, + 0.07404517382383347, + 0.016520503908395767, + 0.04545323923230171, + 0.03656326234340668, + 0.05146809667348862, + 0.02923683449625969, + 0.008360275067389011, + 0.022579140961170197, + 0.01807738095521927, + 0.019519880414009094, + 0.0072213439270854, + 0.016513053327798843, + 0.007634550333023071, + 0.10431190580129623 + ], + "n_pairs": 25 + }, + "PART_WHOLE": { + "best_layer": 19, + "best_head": 13, + "cos_best": 0.14156442880630493, + "cos_2nd": 0.08958695083856583, + "specificity": 1.5801902609813272, + "signal_snr": 2.032056497436996, + "cos_profile": { + "7": 0.074866883456707, + "8": 0.09754882007837296, + "9": 0.052290454506874084, + "10": 0.08257871121168137, + "11": 0.09477108716964722, + "12": 0.1138913631439209, + "13": 0.10907406359910965, + "14": 0.09168244898319244, + "15": 0.08445598185062408, + "16": 0.09252829849720001, + "17": 0.10884886980056763, + "18": 0.10870933532714844, + "19": 0.14156442880630493, + "20": 0.09664797782897949, + "21": 0.10363911092281342, + "22": 0.1103472113609314, + "23": 0.0769500732421875 + }, + "cos_per_head_at_best_layer": [ + 0.06263955682516098, + 0.08958695083856583, + 0.02790617011487484, + 0.04369782656431198, + 0.015710769221186638, + 0.030219461768865585, + 0.04538287967443466, + 0.015228217467665672, + 0.03741630166769028, + 0.0366860032081604, + 0.022136393934488297, + 0.06558110564947128, + 0.006883788853883743, + 0.14156442880630493, + 0.061969924718141556, + 0.052483316510915756 + ], + "n_pairs": 20 + } + } +} \ No newline at end of file diff --git a/data/dict1_primitives/gpt2-medium_dict3_atoms.json b/data/dict1_primitives/gpt2-medium_dict3_atoms.json new file mode 100644 index 0000000000000000000000000000000000000000..3c06537ead3f0f1c65d84b7b6c454dd0fe980e6c --- /dev/null +++ b/data/dict1_primitives/gpt2-medium_dict3_atoms.json @@ -0,0 +1,8303 @@ +{ + "model": "gpt2-medium", + "L_crit": 23, + "mean_cross_cosim": 0.08623496388756176, + "primitives": { + "NEGATION": { + "l_star": 8, + "h_dom": 7, + "snr": 1.9134082078126524, + "specificity": 1.4360684352938828, + "canonical_vec": [ + -0.0017456765053793788, + 0.03816136345267296, + -0.04263561591506004, + -0.08126421272754669, + 0.032588474452495575, + -0.025625038892030716, + 0.004042674787342548, + -0.05757302790880203, + -0.044445011764764786, + 0.00160692329518497, + -0.024371936917304993, + -0.07395146042108536, + 0.009941428899765015, + 0.007789584342390299, + -0.027811648324131966, + 0.05754562094807625, + 0.04653986915946007, + 0.027471376582980156, + 0.006867116782814264, + -0.009475280530750751, + 0.026959093287587166, + -0.03707059100270271, + 0.018298599869012833, + 0.0005428165313787758, + -0.012682432308793068, + -0.03727585822343826, + -0.0296148881316185, + 0.03073420561850071, + -0.009847232140600681, + -0.019710266962647438, + 0.007364705670624971, + 0.0006836160901002586, + 0.03806934505701065, + 0.0007492453441955149, + -0.02768014743924141, + 0.029503149911761284, + 0.009394972585141659, + -0.027055218815803528, + -0.031881388276815414, + -0.0002849062148015946, + 0.006050626747310162, + 0.02467786706984043, + -0.05834299325942993, + 0.021097199991345406, + 0.010064116679131985, + -0.03768946975469589, + 0.002280240412801504, + -0.015607231296598911, + -0.02705560438334942, + -0.003714395919814706, + -0.01768391579389572, + 0.004469219595193863, + 0.04386337846517563, + 0.01389379147440195, + -0.04818585887551308, + -0.015579762868583202, + 0.013729635626077652, + -0.019100934267044067, + 0.004870767239481211, + 0.004505145829170942, + -0.01552989985793829, + 0.013979872688651085, + 0.0181115735322237, + -0.06040717661380768, + -0.010367479175329208, + -0.025937506929039955, + 0.0061307321302592754, + -0.02863968536257744, + -0.02293526381254196, + -0.011160041205585003, + 0.008874207735061646, + 0.01894509233534336, + -0.002528943121433258, + -0.029153896495699883, + -0.057855479419231415, + 0.018363967537879944, + 0.0139958206564188, + -0.04285026341676712, + 0.009552017785608768, + -0.018669769167900085, + -0.030552903190255165, + -0.04040592536330223, + 0.008956401608884335, + 0.03969337418675423, + -0.03422679752111435, + -0.00986874382942915, + 0.006558326072990894, + 0.037023574113845825, + 0.09067124873399734, + 0.05680255964398384, + -0.032534703612327576, + -0.030913881957530975, + -0.02389540523290634, + 0.000564516638405621, + 0.02594912424683571, + -0.047859616577625275, + -0.02681788243353367, + -0.04607013612985611, + 0.004210706800222397, + 0.043741825968027115, + 0.03572952747344971, + 0.03560252860188484, + 0.011205354705452919, + 0.04588939994573593, + 0.0035083789844065905, + 0.01712452806532383, + -0.01806565187871456, + -0.026396848261356354, + 0.02065299078822136, + -0.049231335520744324, + 0.01076405681669712, + 0.020555958151817322, + -0.03053813986480236, + 0.048642516136169434, + -0.003705501090735197, + -0.04732571914792061, + -0.05461983382701874, + 0.027249056845903397, + 0.019947988912463188, + -0.0013587255962193012, + 0.05611506476998329, + 0.007247750647366047, + -0.006866519805043936, + -0.003556429175660014, + -0.0073111336678266525, + 0.024030303582549095, + 0.01008533500134945, + -0.050022874027490616, + 0.05146344378590584, + 0.04223346710205078, + -0.01628538779914379, + -0.02071359194815159, + -0.017872679978609085, + -0.026922238990664482, + 0.03744221851229668, + 0.008537288755178452, + -0.007652871310710907, + 0.047150228172540665, + -0.032210882753133774, + 0.0192644614726305, + -0.03930729627609253, + -0.01860935240983963, + -0.054366275668144226, + 0.01977950893342495, + 0.06815984100103378, + -0.04033870995044708, + -0.012864607386291027, + -0.007479170337319374, + -0.044587504118680954, + -0.011401868425309658, + 0.052264343947172165, + -0.004419540520757437, + -0.009944887831807137, + 0.023720132187008858, + -0.010605741292238235, + 0.018458643928170204, + -0.01767156831920147, + 0.005498250015079975, + 0.0012532116379588842, + -0.06401942670345306, + -0.014797107316553593, + 0.11077263951301575, + -0.08883865922689438, + -0.035032883286476135, + -0.07378745079040527, + -0.00846948754042387, + -0.016648519784212112, + -0.03470253944396973, + 0.07352114468812943, + 0.03845228627324104, + -0.01641993597149849, + 0.028264189139008522, + -0.013628778979182243, + -0.02566772885620594, + 0.009053695946931839, + 0.03348728269338608, + -0.0318584218621254, + -0.015296630561351776, + 0.05462770164012909, + -0.0060374378226697445, + 0.010840808972716331, + 0.0478491373360157, + -0.0005216096178628504, + -0.02888011932373047, + 0.029872532933950424, + 0.028170887380838394, + -0.01864154450595379, + -0.01599656231701374, + -0.001957918982952833, + 0.019577842205762863, + -0.010246911086142063, + 0.014850721694529057, + 0.018809078261256218, + 0.015469970181584358, + 0.030749347060918808, + -0.028063271194696426, + 0.020081203430891037, + 0.05851118266582489, + 0.006764600984752178, + -0.027058517560362816, + -0.04229635000228882, + 0.0016818904550746083, + 0.008683176711201668, + -0.006874389015138149, + -0.014225776307284832, + 0.014619667083024979, + 0.0033948514610528946, + -0.009160113520920277, + 0.03574662283062935, + -0.04481891915202141, + 0.014364350587129593, + -0.005029160995036364, + 0.0065960814245045185, + -0.06301795691251755, + -0.049030426889657974, + -0.03676239028573036, + -0.02353033982217312, + 0.0058612278662621975, + -0.03554465249180794, + -0.004284991882741451, + 0.01429956965148449, + 0.012617313303053379, + 0.011763819493353367, + -0.015726249665021896, + 0.04267769306898117, + 0.014117125421762466, + 0.025459177792072296, + -0.011916648596525192, + 0.03759740665555, + -0.02318718284368515, + 0.03515230491757393, + -0.06647340208292007, + 0.02660028450191021, + 0.009139711037278175, + -0.008764193393290043, + -0.0285344161093235, + -0.024390390142798424, + 0.00765446200966835, + -0.03044659085571766, + -0.01133185625076294, + 0.022131258621811867, + -0.0698745921254158, + 0.022950677201151848, + -0.005411183927208185, + 0.017595840618014336, + 0.011196442879736423, + 0.011464246548712254, + 0.06634119898080826, + -0.0008381079533137381, + 0.015842895954847336, + 0.003444960108026862, + 0.00572369946166873, + 0.002355769043788314, + 0.010122998617589474, + 0.05389867722988129, + -0.041438475251197815, + -0.02706766314804554, + 0.0018585590878501534, + 0.0010572735918685794, + -0.01938062347471714, + -0.011760828085243702, + -0.012580320239067078, + -0.011435606516897678, + -0.0041109113954007626, + 0.0007559386431239545, + 0.0005468224408105016, + -0.010067385621368885, + -0.032194845378398895, + -0.022203899919986725, + -0.05134956166148186, + 0.03606155514717102, + -0.0012447560438886285, + 0.017290420830249786, + -0.015307527966797352, + -0.019838089123368263, + -0.005836421623826027, + -0.002281260211020708, + -0.01693335361778736, + -0.0006171453278511763, + -0.06603740900754929, + 0.02981700748205185, + 0.028902214020490646, + -0.0080243069678545, + 0.012657452374696732, + 0.017800267785787582, + 0.006938897538930178, + 0.022652199491858482, + -0.0005315025919117033, + -0.003525836393237114, + 0.01241945568472147, + -0.03492657467722893, + -0.037948839366436005, + 0.040524739772081375, + 0.004198799375444651, + -0.007894610054790974, + -0.0061661978252232075, + -0.029842566698789597, + -0.017060434445738792, + -0.044648364186286926, + 0.011064503341913223, + -0.007867297157645226, + -0.008211991749703884, + -0.018367376178503036, + 0.0003559807955753058, + -0.03895841911435127, + 0.010146214626729488, + -0.031086008995771408, + 0.03245793655514717, + 0.007546526379883289, + 0.04512111097574234, + 0.0644412562251091, + 0.002524330746382475, + -0.05871446803212166, + -0.006404091138392687, + -0.05557643622159958, + 0.018979014828801155, + -0.024132613092660904, + 0.015000376850366592, + -0.00249708304181695, + 0.007815791293978691, + 0.08132050186395645, + 0.00562940863892436, + 0.011917773634195328, + 0.006069276947528124, + 0.05417347326874733, + 0.028099101036787033, + 0.012472687289118767, + 0.005793374497443438, + 0.04949641227722168, + -0.014174641110002995, + 0.005934231448918581, + -0.03786410763859749, + 0.08010797202587128, + 0.017707837745547295, + -0.0548228994011879, + 0.01319940946996212, + -0.06527881324291229, + -0.0123512651771307, + -0.039455365389585495, + -0.04738462343811989, + -0.05070766806602478, + 0.00786835141479969, + 0.05642153322696686, + 0.011530596762895584, + -0.010300668887794018, + -0.06587568670511246, + 0.02462480403482914, + -0.05519452318549156, + 0.04840214177966118, + -0.033174410462379456, + 0.012117606587707996, + 0.007975967600941658, + -0.013493821956217289, + 0.026041029021143913, + 0.009034400805830956, + -0.004262962378561497, + 0.03775085136294365, + -0.011545790359377861, + 0.036407262086868286, + -0.016104575246572495, + 0.026906777173280716, + 0.06840991228818893, + -0.03623766452074051, + 0.06622566282749176, + 0.01533238124102354, + 0.026989074423909187, + -0.006787802558392286, + 0.0023351002018898726, + 0.01360318809747696, + -0.027413522824645042, + 0.04942506551742554, + 0.0305827297270298, + 0.08108863979578018, + 0.023480674251914024, + 0.0005461213295347989, + -0.0053211902268230915, + -0.03639601916074753, + -0.07354077696800232, + -0.015222103334963322, + -0.04294462129473686, + 0.007830402813851833, + 0.0023941267281770706, + -0.031231239438056946, + 0.005249223206192255, + 0.01985320821404457, + 0.004175032489001751, + 0.06245264783501625, + 0.03025033324956894, + -0.03099142573773861, + -0.005896161776036024, + 0.06532870978116989, + -0.031629402190446854, + 0.01888745091855526, + -0.0031364939641207457, + 0.06687821447849274, + 0.009894631803035736, + -0.01842043735086918, + -0.04307776689529419, + -0.016156798228621483, + -0.044246774166822433, + -0.02767733484506607, + 0.024113979190587997, + -0.06270667165517807, + -0.0015744868433102965, + -0.08739504963159561, + -0.0010380401508882642, + 0.031999099999666214, + 0.02624465897679329, + -0.005605851765722036, + -0.002977980300784111, + -0.02220933325588703, + -0.0037253827322274446, + -0.0074963946826756, + 0.006586022209376097, + -0.0012200713390484452, + 0.043839018791913986, + 0.010343050584197044, + 0.03431333228945732, + 0.0009424182353541255, + -0.017052117735147476, + 0.06433047354221344, + -0.02440798096358776, + -0.04017530381679535, + -0.017880713567137718, + -0.014399921521544456, + -0.009580949321389198, + 0.01638065278530121, + 0.007978394627571106, + -0.024372544139623642, + -0.04905717819929123, + 0.010276752524077892, + -0.02541184239089489, + -0.05800192803144455, + -0.0260391216725111, + 0.03508919104933739, + 0.027163658291101456, + -0.014103117398917675, + -0.018060948699712753, + -0.012468024156987667, + 0.037523120641708374, + -0.01846189796924591, + -0.027470659464597702, + 0.027920132502913475, + 0.054659903049468994, + -0.019042331725358963, + 2.379079523961991e-05, + 0.02774909697473049, + 0.03530009835958481, + -0.048334598541259766, + 0.03213611990213394, + -0.012006528675556183, + 0.012113409116864204, + -0.010037974454462528, + -0.05749112740159035, + 0.004051365423947573, + -0.0026586854364722967, + 0.026740435510873795, + -0.007821030914783478, + 0.021044909954071045, + -0.05472400411963463, + -0.009715580381453037, + -0.0036474501248449087, + 0.02734023705124855, + 0.001145832473412156, + -0.025416528806090355, + 0.022486066445708275, + -0.08216998726129532, + -0.020937351509928703, + 0.04509605094790459, + 0.048233550041913986, + -0.015762735158205032, + -0.007101475726813078, + 0.03260853514075279, + -0.04590180516242981, + 0.008535899221897125, + -0.012591222301125526, + 0.012825536541640759, + -0.03803819790482521, + -0.018752427771687508, + -0.02769412100315094, + 0.024043507874011993, + 0.016169723123311996, + 0.08172095566987991, + 0.011455043219029903, + 0.011061758734285831, + 0.05210574343800545, + 0.000902840867638588, + 0.03933652490377426, + 0.008261102251708508, + 0.019016820937395096, + -0.027513355016708374, + -0.04189624637365341, + -0.04019647464156151, + 0.02099292166531086, + -0.001467978348955512, + -0.03509410470724106, + 0.03671688586473465, + 0.03320919722318649, + -0.010774793103337288, + -0.014267509803175926, + -0.03586897999048233, + -0.03374491631984711, + -0.03247019276022911, + -0.007342011667788029, + -0.018941795453429222, + -0.005741511471569538, + 0.02723507769405842, + 0.009371962398290634, + 0.024457991123199463, + -0.006382445339113474, + -0.01979285106062889, + -0.02091081067919731, + 0.03150703012943268, + -0.01261902041733265, + -0.029205912724137306, + -0.019272252917289734, + 0.0013590982416644692, + 0.0006725724088028073, + 0.008711081929504871, + 0.005249178037047386, + 0.062009356915950775, + 0.018237680196762085, + 0.008672664873301983, + -0.022779226303100586, + -0.006584784481674433, + -0.00903389137238264, + 0.0434875562787056, + 0.03164470195770264, + -0.03245781734585762, + -0.0075142583809792995, + 0.002304030116647482, + -0.0164144579321146, + -0.03982888162136078, + -0.01997668668627739, + -0.010623898357152939, + 0.0009049575310200453, + -0.010998058132827282, + 0.020205432549118996, + -0.023698199540376663, + 0.018938014283776283, + -0.023814700543880463, + 0.028366200625896454, + 0.004518477246165276, + 0.027109503746032715, + 0.03372858092188835, + 0.030204765498638153, + -0.023296738043427467, + -0.011423308402299881, + 0.030765822157263756, + 0.022057805210351944, + -0.0054143210873007774, + -0.04017027094960213, + -0.007530564442276955, + -0.019920403137803078, + -0.06749125570058823, + 0.10186975449323654, + -0.011115980334579945, + -0.03440163657069206, + -0.024744831025600433, + -0.043495651334524155, + 0.02658756822347641, + 0.030741844326257706, + -0.03239854797720909, + -0.022561563178896904, + 0.02133318968117237, + 0.07755079120397568, + -0.03566070646047592, + 0.04035210236907005, + 0.011050906032323837, + -0.04967441409826279, + -0.003916185349225998, + -0.003909400198608637, + -0.015043679624795914, + 0.02326221391558647, + -0.033644139766693115, + -0.03661084547638893, + -0.048632435500621796, + -0.041541676968336105, + -0.011767053045332432, + -0.01753399893641472, + 0.01025247573852539, + -0.00876038521528244, + -0.006936983671039343, + -0.06754879653453827, + -0.02544516697525978, + -0.012830478139221668, + 0.004437427967786789, + -0.03209163248538971, + -0.00538448803126812, + 0.0061489129438996315, + 0.01052297092974186, + 0.016328422352671623, + 0.03715766966342926, + 0.032143160700798035, + -0.053017254918813705, + -0.028168391436338425, + 0.011990663595497608, + 0.02185317687690258, + 0.05559219792485237, + 0.00412312988191843, + -0.008237962611019611, + -0.020431319251656532, + 0.006717546842992306, + 0.015543137677013874, + -0.05298274755477905, + 0.014362928457558155, + 0.02183273807168007, + -0.0023894808255136013, + -0.08283738791942596, + -0.026261616498231888, + 0.05180062726140022, + -0.008023240603506565, + 0.02910052239894867, + -0.0475480891764164, + 0.028286706656217575, + 0.07273657619953156, + 0.009464872069656849, + -0.011566400527954102, + 0.027877938002347946, + -0.03259831294417381, + 0.050732653588056564, + 0.06021607294678688, + -0.02691793628036976, + 0.01227029599249363, + 0.0011626420309767127, + -0.01462572067975998, + 0.015461324714124203, + 0.04770643636584282, + -0.011171036399900913, + -0.02920667454600334, + 0.00192148401401937, + -0.02129039354622364, + -0.010792428627610207, + -0.01926063559949398, + -0.01870918646454811, + 0.005609974730759859, + -0.00420515239238739, + -0.037495825439691544, + 0.01765572279691696, + 0.0070581925101578236, + -0.02568497136235237, + -0.056366100907325745, + -0.059553664177656174, + 0.029028920456767082, + -0.0030553292017430067, + 0.055852338671684265, + 0.069605253636837, + 0.029385961592197418, + 0.017549406737089157, + 0.0002498963731341064, + -0.008454698137938976, + 0.04963451251387596, + 0.0052846139296889305, + -0.01367343682795763, + -0.037187520414590836, + -0.023201676085591316, + -0.046689148992300034, + 0.04356996715068817, + 0.031959399580955505, + -0.022054841741919518, + 0.013759298250079155, + 0.0032751248218119144, + -0.030424952507019043, + -0.039383526891469955, + 0.0028685990255326033, + -0.017900964245200157, + -0.025896310806274414, + 0.03106406144797802, + 0.011685962788760662, + -0.0328233428299427, + 0.03220083937048912, + 0.017605269327759743, + 0.0323331356048584, + -0.01339485589414835, + 0.002042593900114298, + -0.022156549617648125, + -0.05475309118628502, + 0.030237289145588875, + -0.0036644914653152227, + 0.05051015317440033, + 0.07068885862827301, + 0.05987829342484474, + -0.05290672555565834, + -0.013226072303950787, + 0.004529514815658331, + -0.011995463632047176, + -0.012124601751565933, + -0.005992650520056486, + -0.0010890178382396698, + -0.026779334992170334, + 0.0031087619718164206, + 0.04375087097287178, + 0.014647194184362888, + -0.002425705548375845, + -0.012566306628286839, + 0.00892874225974083, + 0.016764936968684196, + -0.03657401725649834, + -0.0010923761874437332, + 0.012013210915029049, + 0.025994841009378433, + 0.01581905223429203, + 0.025610093027353287, + -0.03178007900714874, + -0.05261923372745514, + -0.042016614228487015, + 0.028460247442126274, + -0.02304230071604252, + -0.020666247233748436, + 0.022235678508877754, + 0.07016754895448685, + 0.009660298936069012, + 0.05455172434449196, + 0.024672381579875946, + 0.0649682953953743, + -0.004667699337005615, + 0.017167791724205017, + 0.03397020697593689, + 0.010871621780097485, + 0.03815841302275658, + 0.04317780211567879, + -0.003322185715660453, + -0.006348639726638794, + 0.024100223556160927, + -0.010249401442706585, + 0.011481920257210732, + -0.006266741082072258, + 0.03930903971195221, + -0.01849556341767311, + -0.004467255901545286, + 0.051755864173173904, + -0.0142514668405056, + -0.07273027300834656, + 0.029424523934721947, + -0.0033010749612003565, + 0.04284609109163284, + -0.0461580716073513, + -0.012113494798541069, + -0.02992240898311138, + 0.03159830719232559, + -0.058769918978214264, + 0.059002116322517395, + -0.02363869734108448, + 0.03881300613284111, + -0.0029585256706923246, + 0.08069976419210434, + -0.02733994461596012, + -0.013832691125571728, + -0.009702514857053757, + 0.02202596887946129, + 0.0017609193455427885, + 0.019983835518360138, + 0.008877863176167011, + 0.008089723065495491, + 0.012890256009995937, + -0.021615972742438316, + -0.02041005529463291, + 0.00556174898520112, + 0.05562640354037285, + -0.010904144495725632, + -0.04618651047348976, + 0.029801873490214348, + -0.021141355857253075, + 0.002373548224568367, + -0.03068201057612896, + -0.029742084443569183, + -0.024700097739696503, + -0.004851713310927153, + 0.010382234118878841, + 0.016939852386713028, + 0.015799209475517273, + -0.02342156134545803, + 0.0034367733169347048, + 0.011972718872129917, + 0.007725472096353769, + -0.0019231510814279318, + 0.024644415825605392, + -0.05604933202266693, + -0.04070879518985748, + 0.002643182873725891, + 0.023460103198885918, + -0.012251906096935272, + 0.024539748206734657, + -0.02345011755824089, + -0.03132809326052666, + -0.012141135521233082, + -0.004980544093996286, + 0.004253632854670286, + 0.025999872013926506, + 0.052851252257823944, + 0.06178869307041168, + 0.03427782654762268, + 0.0021000367123633623, + -0.02914397418498993, + -0.054173387587070465, + -0.05293381214141846, + 0.04843178018927574, + 0.022215748205780983, + -0.00273641268722713, + -0.025248853489756584, + 0.00873282365500927, + -0.03279658406972885, + 0.03270704671740532, + 0.027899134904146194, + 0.04459986090660095, + 0.045300018042325974, + 0.02808436192572117, + 0.006605937611311674, + 0.006475445348769426, + -0.02847280167043209, + -0.03237421065568924, + 0.002182859228923917, + -0.013987746089696884, + 0.060588981956243515, + -0.012535087764263153, + 0.025272300466895103, + -0.0014038837980479002, + -0.03711862117052078, + -0.005233666859567165, + 0.03890222683548927, + 0.02834378369152546, + 0.011558720842003822, + -0.03384900465607643, + 0.04383324831724167, + 0.03823716938495636, + 0.002947266446426511, + 0.030414897948503494, + 0.03273376449942589, + -0.0532676987349987, + -0.05536917224526405, + 0.03320717439055443, + 0.08332453668117523, + 0.012037716805934906, + 0.025579743087291718, + 0.033460844308137894, + -0.007699969224631786, + -0.02835402823984623, + 0.02914126217365265, + -0.021797506138682365, + -0.068535216152668, + 0.014329055324196815, + -0.03407913073897362, + 0.011074401438236237, + -0.03993484005331993, + 0.009769706055521965, + 0.02632046304643154, + 0.007299251388758421, + 0.039343807846307755, + -0.00165578443557024, + 0.02610476315021515, + 0.0034754739608615637, + 0.012202119454741478, + 0.005047410260885954, + 0.029591985046863556, + -0.038720231503248215, + -0.0019159591756761074, + 0.03041369467973709, + 0.048319436609745026, + 0.03193502128124237, + 0.0093455258756876, + 0.07936421781778336, + 0.017456624656915665, + -0.0054725841619074345, + 0.007151291240006685, + 0.005708872806280851, + 0.017338842153549194, + 0.008604212664067745, + -0.00468292785808444, + -0.05012943223118782, + 0.0012371779885143042, + 0.0037727071903645992, + -0.012603622861206532, + 0.010737559758126736, + 0.0397091880440712, + -0.01451350748538971, + 0.022486872971057892, + 0.00609981082379818, + -0.05853668972849846, + 0.00941945519298315, + -0.060293734073638916, + -0.010193453170359135, + -0.025313830003142357, + -0.016589749604463577, + -0.05168187990784645, + 0.001980355242267251, + -0.020221000537276268, + -0.006908762734383345, + -0.04388214275240898, + -0.058439165353775024, + -0.022259552031755447, + -0.006334207020699978, + -0.0742516815662384, + -0.055486731231212616, + -0.012447388842701912, + -0.035876333713531494, + 0.024415751919150352, + 0.01585661619901657, + 0.013094871304929256, + -0.08363966643810272, + 0.0016400336753576994, + -0.003879483789205551, + -0.01940152235329151, + 0.027214383706450462, + 0.0590699128806591, + -0.00010691316128941253, + -0.027076104655861855, + 0.0241018608212471, + -0.005652585066854954, + -0.019949430599808693, + -0.004108302760869265, + -0.014374209567904472, + 0.004598809871822596, + 0.00017063628183677793, + -0.020014911890029907, + -0.029277481138706207, + 0.00039962262962944806, + -0.0008673174888826907, + -0.02339898608624935, + -0.002859752392396331, + 0.023821327835321426, + -0.018628152087330818, + 0.006919133476912975, + 0.00791944470256567, + 0.030441870912909508, + -0.03237307444214821, + 0.008190653286874294, + 0.044456690549850464, + 0.017464138567447662, + 0.04587605595588684, + -0.05185063183307648, + -0.0097061637789011, + -0.016220765188336372, + 0.030267314985394478, + 0.04461683705449104, + -0.007214597426354885, + 0.01694393716752529, + -0.016479630023241043, + -0.027511300519108772, + 0.04327097162604332, + -0.020657500252127647, + 0.0031464193016290665, + 0.01027834415435791, + 0.009574957191944122, + -0.026527274399995804, + -0.021474173292517662, + -0.08973786234855652, + -0.002797611290588975, + 0.07474735379219055, + 0.030205758288502693, + -0.01743125729262829, + 0.022217821329832077, + 0.022502712905406952, + 0.025577601045370102, + -0.02875254862010479, + -0.0048540858551859856, + -0.049076855182647705, + 0.020154565572738647, + 0.007970665581524372, + -0.016401592642068863, + 0.028651811182498932, + -0.04082673788070679, + -0.004871581681072712, + 0.037566617131233215, + 0.02269815281033516, + 0.027526479214429855, + 0.014711400493979454, + -0.02380496636033058, + 0.01779550313949585, + 0.030807986855506897, + 0.009463912807404995, + -0.0212149228900671, + 0.06507014483213425, + -0.02553391456604004, + -0.034575238823890686, + -0.008983063511550426, + -0.04108484834432602, + 0.07109629362821579, + 0.007709113880991936, + 0.03578894957900047, + 0.0027516225818544626, + -0.01758607290685177, + -0.01681741699576378, + 0.013524558395147324, + -0.010940075851976871, + 0.023946572095155716, + 0.023815354332327843, + 0.008253942243754864, + -0.013584553264081478, + -0.03554743155837059, + 0.02184642106294632, + 0.00634699547663331, + 0.02851424552500248, + -0.004023163113743067, + -0.016877152025699615, + 0.01658087596297264, + -0.005269027315080166, + 0.0038188956677913666, + 0.012744664214551449, + -0.025661416351795197, + 0.005303733516484499, + 0.016346657648682594, + -0.006726055406033993, + -0.07989314198493958, + 0.04171900823712349, + -0.0032808270771056414, + -0.02065904811024666, + -0.005752177909016609, + 0.00821492075920105, + 0.030267247930169106, + -0.005537765100598335, + 0.033869437873363495, + -0.013603128492832184, + 0.029857708141207695, + 0.0361659862101078, + 0.013156049884855747, + -0.010691719129681587, + 0.05582300201058388, + 0.01616305112838745, + 0.0338049978017807, + 0.019199417904019356, + 0.028893280774354935, + -0.020858243107795715, + 0.00518115796148777, + 0.023955315351486206 + ], + "attn_peak_rel": -1.875, + "attn_entropy": 0.38197257930277184, + "attn_spread_pm2": 0.5809112805873156, + "attn_role": "looks_back", + "n_pairs": 8 + }, + "CAUSATION": { + "l_star": 16, + "h_dom": 15, + "snr": 2.29386122885596, + "specificity": 1.6682694982992246, + "canonical_vec": [ + 0.03402306139469147, + 0.010190105997025967, + 0.010749905370175838, + -0.0009199745254591107, + 0.0735868439078331, + 0.059920091181993484, + 0.008461483754217625, + 0.07816502451896667, + 0.015256538987159729, + 0.03817412257194519, + -0.04744861274957657, + 0.06557635962963104, + 0.020265402272343636, + 0.05071917548775673, + 0.018831904977560043, + 0.044275689870119095, + 0.01715776138007641, + -0.01156131736934185, + -0.04993858188390732, + -0.034151796251535416, + -0.05624327436089516, + 0.08222835510969162, + 0.031855080276727676, + -0.012588419951498508, + -0.06249741092324257, + -0.006797413807362318, + 0.005195752251893282, + -0.06889300048351288, + -0.014690305106341839, + 0.029990119859576225, + 0.009147468023002148, + 0.012567208148539066, + -0.02520572952926159, + 0.029570119455456734, + -0.025779563933610916, + -0.012242461554706097, + -0.01079265121370554, + 0.029801232740283012, + -0.012936796061694622, + -0.040604494512081146, + 0.05777432769536972, + -0.04345989227294922, + -0.03124786540865898, + 0.027587179094552994, + 0.008575288578867912, + 0.03356296196579933, + -0.0799388438463211, + 0.0355176217854023, + 0.012093630619347095, + 0.013666513375937939, + 0.018207823857665062, + -0.022178800776600838, + -0.008661343716084957, + 0.030437294393777847, + 0.05012379214167595, + 0.03908209502696991, + 0.03949180990457535, + -0.0016002926276996732, + 0.014745709486305714, + -0.003639067756012082, + -0.002850235439836979, + 0.02708400972187519, + 0.0014620372094213963, + 0.013591654598712921, + 0.04881497845053673, + 0.0166107676923275, + -0.0325189046561718, + 0.03783159330487251, + -0.009733552113175392, + 0.06046798825263977, + 0.007690009195357561, + -0.004317726008594036, + -0.015743067488074303, + -0.02289627119898796, + -0.0015704177785664797, + -0.04514104872941971, + 0.013458549045026302, + 0.07914957404136658, + 0.009408433921635151, + 0.026936164125800133, + -0.010104069486260414, + 0.07840569317340851, + 0.01985071413218975, + 0.017381176352500916, + 0.015121879987418652, + 0.07591156661510468, + 0.021728139370679855, + -0.006860415451228619, + -0.014011750929057598, + 0.00858173705637455, + 0.015823686495423317, + 0.05319969356060028, + 0.011371253058314323, + 0.031996652483940125, + 0.06448332220315933, + -0.05447179824113846, + -0.02391316182911396, + -0.0269667636603117, + -0.02113822102546692, + 0.04537629708647728, + 0.019067488610744476, + -0.009653179906308651, + 0.019499946385622025, + -0.005923429038375616, + -0.0301327146589756, + 0.025137970224022865, + -0.04860787093639374, + -0.00610287394374609, + 0.04562048241496086, + -0.019861070439219475, + 0.002956557087600231, + -0.054547566920518875, + 0.03291117399930954, + -0.038891181349754333, + 0.0015228380216285586, + 0.0004977163625881076, + -0.04042461887001991, + -0.034342918545007706, + 0.06759587675333023, + 0.014629213139414787, + -0.0562170185148716, + -0.006152081768959761, + -0.009670835919678211, + 0.005589580163359642, + -0.10320127755403519, + -0.021615231409668922, + 0.0034045649226754904, + 0.011785745620727539, + -0.022150106728076935, + 0.013491148129105568, + -0.01098951417952776, + -0.019374987110495567, + -0.03521913290023804, + 0.05167586728930473, + 0.03899848833680153, + 0.004037950653582811, + 0.0012999795144423842, + 0.005720216780900955, + -0.07068917155265808, + 0.04119141399860382, + -0.0006835981039330363, + 0.03582241013646126, + -0.005001585930585861, + 0.0336676761507988, + 0.07546538859605789, + 0.0049512009136378765, + -0.015099343843758106, + 0.05283178389072418, + -0.01942513883113861, + 0.004323715344071388, + 0.012829581275582314, + 0.015547572635114193, + 0.02282765507698059, + 0.008202137425541878, + -0.011442826129496098, + 0.025896789506077766, + 0.03988909721374512, + 0.017160790041089058, + 0.05904897302389145, + 0.001077865599654615, + -0.02075246348977089, + -0.017289990559220314, + 0.015785058960318565, + 0.002622065134346485, + 0.005112800747156143, + 0.020760193467140198, + 0.006822772324085236, + 0.0018378148088231683, + 0.033321164548397064, + 0.04564620181918144, + 0.023683760315179825, + 0.003545789048075676, + 0.005486429203301668, + -0.025957953184843063, + 0.035433314740657806, + 0.011268229223787785, + -0.004180344752967358, + 0.027044901624321938, + -0.03914365917444229, + -0.02061973512172699, + -0.034122928977012634, + 0.0033885855227708817, + -0.06547827273607254, + -0.025076013058423996, + 0.0345265157520771, + -0.01136512216180563, + -0.020879369229078293, + 0.03847118094563484, + -0.0034316887613385916, + 0.006822558119893074, + 0.01092153787612915, + 0.018980415537953377, + -0.019235342741012573, + 0.049020055681467056, + -0.017575468868017197, + -0.01867479644715786, + -0.0269785076379776, + 0.020374037325382233, + 0.0031061149202287197, + -0.016438554972410202, + -0.00915967021137476, + 0.031427137553691864, + 0.010186459869146347, + -0.02114212140440941, + 0.026337383314967155, + 0.02105865627527237, + -0.0075247930362820625, + 0.0027855972293764353, + 0.030047018080949783, + -0.03490237891674042, + 0.021953171119093895, + 0.004413253162056208, + -0.0074485838413238525, + -0.04014232009649277, + -0.058996688574552536, + -0.03279179334640503, + 0.015865527093410492, + -0.005848901346325874, + -0.028002124279737473, + 0.04787924885749817, + 0.05220089852809906, + 0.012522023171186447, + -0.033263638615608215, + 0.030517367646098137, + 0.007631123997271061, + 0.016713475808501244, + 0.006345649715512991, + 0.015072384849190712, + -0.04558984562754631, + -0.0228568185120821, + -0.0007667490863241255, + -0.005503435153514147, + -0.049235858023166656, + 0.023982955142855644, + 0.015898915007710457, + 0.06623825430870056, + -0.04532640054821968, + -0.01733534038066864, + 0.004478054586797953, + -0.03515685722231865, + 0.024014651775360107, + -0.02977500855922699, + -0.01613808237016201, + -0.05519799515604973, + 0.0006591601995751262, + 0.008701848797500134, + -0.010119972750544548, + 0.0016907111275941133, + 0.028303777799010277, + -0.023269228637218475, + -0.018930908292531967, + -0.044692955911159515, + -0.05453554540872574, + -0.015063316561281681, + 0.04010755568742752, + -0.06570512056350708, + 0.042476117610931396, + -0.007732231169939041, + -0.03842909634113312, + 0.011609750799834728, + 0.045505404472351074, + -0.02704469859600067, + 0.028638716787099838, + 0.004544591531157494, + -0.01576545089483261, + 0.11070746928453445, + 0.06075636297464371, + 0.022605406120419502, + 0.24461181461811066, + -0.008582388050854206, + -0.002232258440926671, + 0.003286634339019656, + 0.03266831859946251, + 0.027150290086865425, + -0.05832428112626076, + 0.012440389953553677, + 0.022340623661875725, + -0.0050589703023433685, + -0.016328398138284683, + -0.017263520509004593, + -0.009549451991915703, + 0.06167829781770706, + -0.008832812309265137, + -0.005503238178789616, + -0.041211508214473724, + 0.034546978771686554, + -0.04923135042190552, + -0.029817357659339905, + 0.0031950525008141994, + 0.023077314719557762, + 0.016879502683877945, + -0.008162206970155239, + 0.007734604645520449, + -0.01761481910943985, + -0.010714288800954819, + -0.0136696333065629, + 0.009353281930088997, + 0.01066782046109438, + -0.02427748218178749, + 0.02863546460866928, + 0.05126005783677101, + 0.003041005227714777, + 0.02920057624578476, + 0.03389081731438637, + -0.01237356849014759, + -0.007979382760822773, + -0.009355870075523853, + 0.029978616163134575, + -0.04291468858718872, + 0.013860369101166725, + -0.03236202150583267, + 0.03153524547815323, + 0.008116264827549458, + 0.012314334511756897, + -0.02254047431051731, + 0.030427327379584312, + -0.03701595962047577, + -0.0430506207048893, + -0.010558768175542355, + -0.020115582272410393, + -0.0001816318545024842, + -0.015443967655301094, + -0.027296649292111397, + -0.034257400780916214, + -0.02504400722682476, + -0.015940045937895775, + 0.03175274282693863, + -0.03884276747703552, + 0.03714103624224663, + -0.019355298951268196, + -0.01968865841627121, + -0.031193042173981667, + -0.03775746002793312, + 0.018562523648142815, + -0.02434222772717476, + -0.018764324486255646, + -0.025383444502949715, + -0.0028041191399097443, + -0.015324638225138187, + 0.006125153973698616, + 0.019023945555090904, + -0.01021327544003725, + -0.023327110335230827, + -0.04142176732420921, + -0.032154377549886703, + 0.012606381438672543, + 0.03380255028605461, + 0.010294166393578053, + -0.006782046519219875, + -0.012893992476165295, + 0.0030319918878376484, + -0.05295636132359505, + -0.043287500739097595, + 0.0071434625424444675, + 0.042347561568021774, + -0.03421162813901901, + -0.017411155626177788, + 0.03688172623515129, + -0.06073929741978645, + 0.0030952943488955498, + 0.004173870664089918, + -0.004936276935040951, + 0.000718108844012022, + 0.025315256789326668, + 0.042167723178863525, + 0.020421672612428665, + -0.05407285317778587, + -0.008240188471972942, + 0.03577318415045738, + -0.019818561151623726, + -0.0040520490147173405, + -0.01062629371881485, + -0.0003146895032841712, + 0.021418944001197815, + 0.04404503107070923, + -0.09014440327882767, + -0.03142339736223221, + 0.007657914888113737, + -0.01492153201252222, + 0.009450863115489483, + -0.010105921886861324, + 0.0479353703558445, + 0.002161480486392975, + 0.042232852429151535, + 0.024615131318569183, + -0.006862330250442028, + 0.004448730032891035, + -0.03367975726723671, + 0.02463703602552414, + 0.03168461471796036, + -0.04930904135107994, + -0.0022971206344664097, + 0.01836334355175495, + 0.014577423222362995, + 0.004220748320221901, + -0.023298315703868866, + -0.01965431310236454, + 0.03834874927997589, + 0.0411958321928978, + 0.02655136212706566, + 0.027611590921878815, + -0.0370938703417778, + 0.01812138967216015, + -0.002488264814019203, + 0.05499212071299553, + 0.0478510819375515, + -0.010712848044931889, + -6.570393907168182e-06, + 0.005341320764273405, + 0.01505910325795412, + 0.004653077572584152, + -0.01120658777654171, + -0.03315108269453049, + 0.07450570911169052, + 0.020475031808018684, + -0.04598842188715935, + 0.01459056232124567, + -0.001897790003567934, + 0.023938506841659546, + -0.0339803472161293, + 0.061440084129571915, + 0.017825916409492493, + -0.012717966921627522, + -0.03175931051373482, + 0.011777935549616814, + 0.016635030508041382, + -0.02613188698887825, + -0.04298941791057587, + -0.026236308738589287, + 0.0026772390119731426, + -0.024100158363580704, + 0.040101781487464905, + -0.008118562400341034, + -0.05416339635848999, + -0.015237138606607914, + 0.004861400928348303, + -0.021457521244883537, + 0.02397601678967476, + 0.0044399830512702465, + -0.004323930013924837, + 0.026059266179800034, + -0.04081054404377937, + 0.039764467626810074, + -0.04342426732182503, + 0.01655491441488266, + -0.028176626190543175, + 0.006311794742941856, + -0.002766741206869483, + -0.012160871177911758, + 0.009081698022782803, + 0.015465660952031612, + -0.031199850142002106, + -0.026410408318042755, + -0.001191957970149815, + -0.007540750317275524, + 0.023150727152824402, + 0.015578541904687881, + 0.0042380597442388535, + -0.015191387385129929, + -0.01202970091253519, + 0.007227740716189146, + 0.01858518458902836, + -0.01652972400188446, + 0.014264899305999279, + -0.00021441194985527545, + -0.02655821666121483, + -0.02186998538672924, + 0.0038034962490200996, + -0.03703061863780022, + -0.09238854050636292, + -0.008172924630343914, + -0.03652772679924965, + 0.07102123647928238, + -0.027045106515288353, + 0.01861705258488655, + 0.025570232421159744, + 0.06197601184248924, + -0.04858376458287239, + 0.0008576097898185253, + -0.013407943770289421, + 0.03358091413974762, + -0.03833872079849243, + -0.05660903453826904, + -0.04108171537518501, + 0.05298536270856857, + 0.01405083853751421, + -0.013453246094286442, + 0.015589569695293903, + -0.04145268350839615, + -0.04967735335230827, + -0.008156299591064453, + -0.010799572803080082, + -0.04631420224905014, + 0.004566389601677656, + 0.027818987146019936, + -0.010228274390101433, + 0.017931753769516945, + 0.025740372017025948, + -0.012596159242093563, + -0.008795957081019878, + -0.008336366154253483, + -0.029954183846712112, + -0.010134482756257057, + -0.015866704285144806, + -0.021603284403681755, + -0.04812285676598549, + -0.022771406918764114, + 0.028946643695235252, + 0.07439078390598297, + 0.0322660356760025, + -0.04379492253065109, + 0.013387187384068966, + 0.03603614121675491, + 0.038014061748981476, + 0.014552779495716095, + 0.02269025705754757, + 0.026062186807394028, + 0.021767957136034966, + -0.009548910893499851, + 0.003255654824897647, + -0.05041832849383354, + 0.04616445302963257, + -0.003969667479395866, + -0.010137438774108887, + 0.0013770918594673276, + -0.015583495609462261, + -0.05923802778124809, + 0.040580566972494125, + 0.0026925774291157722, + 0.015613838098943233, + -0.011421868577599525, + -0.015487599186599255, + -0.0029863694217056036, + 0.009113645181059837, + -0.005264310631901026, + -0.010112082585692406, + 0.029915135353803635, + 0.007160989101976156, + 0.04936998710036278, + 0.02242117188870907, + -0.022985262796282768, + -0.030938317999243736, + 0.004371279384940863, + -0.03140094876289368, + 0.005996923428028822, + 0.021507862955331802, + -0.034865736961364746, + -0.0566440187394619, + -0.02255880832672119, + -0.006619754247367382, + 0.06202653795480728, + -0.001304621691815555, + 0.030188731849193573, + 0.038523994386196136, + -0.004311488009989262, + 0.030328523367643356, + 0.02252328395843506, + -0.0054880231618881226, + -0.03520667552947998, + -0.008167242631316185, + 0.007464672438800335, + -0.015560082159936428, + -0.012984250672161579, + 0.010013967752456665, + 0.009805234149098396, + 0.00320602860301733, + 0.02934764139354229, + 0.03458511456847191, + -0.023270484060049057, + 0.04442005977034569, + -0.0019019756000488997, + -0.026670217514038086, + -0.0066690887324512005, + 0.012486511841416359, + -0.006567642092704773, + 0.014106076210737228, + 0.011602340266108513, + -0.011217289604246616, + 0.02135966718196869, + -0.03739446774125099, + -0.019307397305965424, + 0.023241370916366577, + 0.00743040069937706, + -0.008332972414791584, + -0.004735575057566166, + 0.0006559501634910703, + -0.02489250898361206, + -0.041384462267160416, + 0.018493715673685074, + 0.05263501778244972, + -0.16671572625637054, + 0.0461101233959198, + -0.0012910071527585387, + -0.029751500114798546, + 0.0063811298459768295, + -0.02077864110469818, + -0.00028381316224113107, + -0.032381556928157806, + -0.023695508018136024, + -0.0008976207464002073, + -0.024458715692162514, + 0.03917867690324783, + -0.019985556602478027, + -0.027749070897698402, + -0.01215165201574564, + -0.0076400358229875565, + 0.024502910673618317, + -0.0009253608295693994, + -0.028971627354621887, + -0.007110703270882368, + 0.005674632266163826, + -0.008100965991616249, + 0.04290793836116791, + -0.009440955705940723, + 0.0267791748046875, + 0.04705082252621651, + -0.025941111147403717, + 0.04499048367142677, + 0.021543268114328384, + -0.002592024626210332, + 0.01314480323344469, + 0.02231132797896862, + -0.03422543779015541, + 0.02155723050236702, + -0.000534695282112807, + 0.019944416359066963, + -0.003426783485338092, + 0.02255709283053875, + -0.016632674261927605, + -0.019366126507520676, + -0.03883030265569687, + 0.04117772728204727, + -0.026524359360337257, + 0.005318482872098684, + 0.005597100593149662, + 0.014356003142893314, + 0.051759690046310425, + 0.03291952237486839, + 0.0034420022275298834, + -0.02569962479174137, + 0.005121850408613682, + -0.02620062045753002, + -0.03530405834317207, + 0.019292248412966728, + 0.010132957249879837, + -0.004462784621864557, + -0.056880027055740356, + -0.021984146907925606, + -0.0037504471838474274, + -0.01238334085792303, + -0.012095075100660324, + -0.02924581617116928, + -0.018823444843292236, + -0.0005762995569966733, + 0.05318700149655342, + -0.04030793532729149, + 0.009812244214117527, + 0.01038407627493143, + -0.03452642634510994, + 0.03240451216697693, + -0.004199301823973656, + -0.0040361229330301285, + 0.00355613068677485, + -0.0011897928779944777, + -0.05226738378405571, + 0.04182594269514084, + 0.017921600490808487, + -0.02410280704498291, + -0.0018245060928165913, + -0.0012401093263179064, + -0.008541235700249672, + -0.013421539217233658, + -0.023356163874268532, + -0.020508743822574615, + -0.023606745526194572, + -0.04706242308020592, + 0.026033449918031693, + -0.013473661616444588, + -0.03433229774236679, + -0.036343470215797424, + 0.005440389271825552, + -0.01742759719491005, + -0.012299571186304092, + -0.008534848690032959, + -0.02897031418979168, + 0.0016741218278184533, + -0.034929290413856506, + -0.022406086325645447, + 0.019888561218976974, + -0.03447715565562248, + -0.028485802933573723, + -0.012877633795142174, + -0.007190396077930927, + 0.02809310331940651, + -0.0008031541365198791, + -0.00592279527336359, + -0.012539817951619625, + 0.0019910079427063465, + -0.00018522788013797253, + 0.0007083730888552964, + 0.02757073938846588, + -0.025393405929207802, + 0.051689352840185165, + -0.008547811768949032, + 0.0056672478094697, + -0.04222195968031883, + -0.028604941442608833, + -0.0433143675327301, + 0.0025976235046982765, + 0.023280806839466095, + -0.00997076090425253, + -0.009842447005212307, + -0.01934347301721573, + -0.04250418022274971, + -0.014694389887154102, + 0.010346620343625546, + 0.0029967855662107468, + -0.00938242394477129, + 0.06378227472305298, + 0.021677622571587563, + 0.008696009404957294, + -0.040333423763513565, + -0.018637899309396744, + 0.029894553124904633, + -0.006151162553578615, + 0.002435914473608136, + 0.03220497816801071, + 0.03910200297832489, + -0.004206981044262648, + -0.04355809837579727, + 0.05289124697446823, + 0.01815091073513031, + 0.0187608040869236, + 0.008259438909590244, + 0.00029164491570554674, + -0.006513913627713919, + 0.029336705803871155, + 0.003906266763806343, + 0.009370868094265461, + -0.01218357589095831, + 0.030321352183818817, + -0.01209714449942112, + -0.00017489508900325745, + -0.016527006402611732, + 0.05200259014964104, + 0.04153364896774292, + 0.0013918800977990031, + 0.03276846185326576, + 0.0006208776030689478, + -0.011787019670009613, + 0.0011287383968010545, + 0.01988804154098034, + 0.010313829407095909, + -0.007064178120344877, + -0.047669317573308945, + 0.01248916331678629, + 0.007289586588740349, + 0.03927793726325035, + -0.03212329000234604, + 0.004958522971719503, + -0.026041023433208466, + -0.006737350020557642, + -0.007033678237348795, + 0.009054599329829216, + 0.028694424778223038, + -0.01799841597676277, + -0.0017975139198824763, + -0.04015873000025749, + -0.02495359629392624, + -0.01180226169526577, + 0.01731615699827671, + 0.020087238401174545, + -0.03985676169395447, + -0.0393904410302639, + -0.004331630188971758, + 0.025108331814408302, + 0.012823824770748615, + -0.0031867048237472773, + 0.008933140896260738, + -0.0019385250052437186, + 0.03411756455898285, + -0.0595182366669178, + -0.0057250759564340115, + -0.0213882215321064, + 0.004787964280694723, + -0.05095918849110603, + -0.019425395876169205, + 0.016045302152633667, + -0.015271668322384357, + 0.022074013948440552, + -0.051709651947021484, + 0.033226512372493744, + 0.0049263243563473225, + -0.06523114442825317, + 0.015141843818128109, + 0.0617876835167408, + 0.07455417513847351, + -0.0656016618013382, + -0.018842089921236038, + 0.028890501707792282, + 0.005974356550723314, + -0.0058472915552556515, + -0.01406995952129364, + -0.005796544253826141, + 0.030631326138973236, + -0.030986225232481956, + -0.00500113982707262, + -0.018536444753408432, + 0.04729532450437546, + 0.04598066955804825, + -0.007278270088136196, + -0.008974927477538586, + -0.000964113452937454, + 0.008226040750741959, + 0.026088841259479523, + 0.013079328462481499, + 0.018374834209680557, + -0.04938580468297005, + 0.02308824099600315, + -0.0005678731831721961, + -0.02293606661260128, + -0.025279846042394638, + -0.012920064851641655, + -0.016909077763557434, + -0.04986231029033661, + -0.00961835216730833, + 0.014952055178582668, + 0.0060540977865457535, + 0.035243961960077286, + 0.02248777635395527, + -0.014720243401825428, + 0.01651807315647602, + 0.011293536052107811, + -0.008523539640009403, + -0.03021891787648201, + 0.0026485545095056295, + -0.08638021349906921, + 0.003145817667245865, + 0.01922658644616604, + -0.00596617953851819, + 0.013700596988201141, + -0.017160343006253242, + 0.009583041071891785, + -0.024714792147278786, + -0.04095057025551796, + -0.017017003148794174, + 0.007844354026019573, + 0.016402510926127434, + -0.03056366927921772, + -0.02377287484705448, + 0.050901658833026886, + 0.018401332199573517, + -0.026050906628370285, + 0.03780384734272957, + -0.00903088878840208, + 0.05155201256275177, + 0.016014140099287033, + 0.0773901641368866, + -0.020504597574472427, + -0.03420323133468628, + -0.03798342123627663, + -0.009533874690532684, + -0.04252910614013672, + 0.06073518469929695, + -0.003452413948252797, + -0.010414427146315575, + -0.004828928504139185, + 0.029462866485118866, + 0.0018933345563709736, + 0.03913361206650734, + -0.013873888179659843, + -0.023200763389468193, + -0.03762924298644066, + -0.020743772387504578, + 0.026248877868056297, + 0.02733614109456539, + 0.02769339643418789, + 0.044910356402397156, + 0.002960786921903491, + 0.027695542201399803, + -0.02186421863734722, + -0.007645508274435997, + -0.04593060910701752, + 0.025771616026759148, + -0.04694657772779465, + 0.007501126732677221, + 0.01326511800289154, + 0.05125245824456215, + -0.005608897656202316, + 0.09804331511259079, + -0.009780450724065304, + 0.01940302550792694, + -0.006908058654516935, + -0.03213537484407425, + 0.0077276937663555145, + 0.000649915193207562, + 0.028714286163449287, + -0.009077533148229122, + 0.008057849481701851, + -0.018161701038479805, + -0.02736726962029934, + 0.0335235670208931, + 0.032840337604284286, + 0.08459825813770294, + -0.037211108952760696, + -0.07730243355035782, + -0.0023200344294309616, + -0.013035752810537815, + 0.024974850937724113, + -0.11117836833000183, + 0.003121529007330537, + -0.013698760420084, + 0.008189809508621693, + -0.014710809104144573, + -0.042828772217035294, + 0.010087423957884312, + -0.03332544490695, + -0.04008304327726364, + -0.06516054272651672, + -0.005969512742012739, + 0.025931842625141144, + -0.015379912219941616, + 0.007207075133919716, + -0.004432516172528267, + -0.008790905587375164, + -0.008949642069637775, + -0.03532631695270538, + 0.00817649345844984, + 0.013647127896547318, + -0.051824793219566345, + -0.00875805877149105, + 0.04136113077402115, + 0.033649228513240814, + 0.015503107570111752, + -0.026063505560159683, + -0.04136122390627861, + -0.03552419692277908, + -0.005002114921808243, + -0.004840416833758354, + -0.01437727827578783, + 0.057858966290950775, + 0.02033253386616707, + -0.027730301022529602, + -0.013124154880642891, + 0.05224129930138588, + 0.011978894472122192, + -0.002926650457084179, + -0.04281745105981827, + 0.02089230716228485, + -0.01210662443190813, + 0.014821494929492474, + 0.016380837187170982, + 0.012303085066378117, + 0.0030897471588104963, + 0.010401899926364422, + 0.030649956315755844, + 0.039290815591812134, + -0.019007472321391106, + -0.007948094047605991, + 0.0019217882072553039, + -0.0067328717559576035, + 0.004985416773706675, + 0.003884251695126295, + -0.0061123669147491455, + 0.04442329704761505, + 0.01591085083782673, + 0.04256317391991615, + 0.04151904210448265, + 0.0046116383746266365, + 0.05142194777727127, + 0.030375761911273003, + -0.018309997394680977, + 0.020380845293402672, + 0.0102331368252635, + 0.04120064154267311, + -0.02219819650053978, + -0.03078470006585121, + -0.02746637538075447, + -0.017929766327142715, + -0.05121465027332306, + -0.02263672649860382, + -0.05816332623362541, + -0.024423852562904358, + 0.06052945926785469, + -0.004634527955204248, + 0.010292790830135345, + -0.04689522832632065, + 0.007426684722304344, + -0.007822588086128235, + 0.01205727830529213, + -0.02589869685471058, + -0.003202730091288686, + 0.030725179240107536, + -0.032007742673158646, + 0.03161807730793953, + 0.00932460930198431, + 0.0015509864315390587, + -0.012926806695759296, + 0.07507690787315369, + -0.0526064932346344, + 0.027101732790470123, + -0.007963879033923149, + -0.005214569624513388, + 0.024643586948513985, + -0.010323450900614262, + -0.028478875756263733, + -0.03313134238123894, + -0.04370925948023796, + -0.00868939608335495, + 0.01432834379374981, + 0.008242969401180744, + -0.04247158765792847, + 0.014624644070863724, + 0.032811153680086136, + -0.009632769972085953, + -0.02170182764530182, + 0.012128881178796291, + -0.028174618259072304, + -0.029203515499830246, + -0.04190978780388832, + 0.0020495704375207424, + 0.026096094399690628, + -2.9836903195246123e-05, + 0.0069220103323459625 + ], + "attn_peak_rel": -2.0, + "attn_entropy": 0.08842310321051627, + "attn_spread_pm2": 0.8754627906309906, + "attn_role": "looks_back", + "n_pairs": 8 + }, + "TEMPORAL_BEFORE": { + "l_star": 20, + "h_dom": 6, + "snr": 2.198647549634198, + "specificity": 1.4885990090511252, + "canonical_vec": [ + 0.034270692616701126, + -0.06077250838279724, + 0.01975814439356327, + -0.030301062390208244, + -0.0042431168258190155, + -0.04743408411741257, + 0.01886793226003647, + -0.04021798074245453, + 0.01266392506659031, + -0.019154256209731102, + -0.021263647824525833, + -0.014954961836338043, + -0.07649152725934982, + 0.013932512141764164, + -0.0015400848351418972, + 0.03297927975654602, + -0.007247851695865393, + 0.039057765156030655, + 0.023775022476911545, + 0.03580962121486664, + 0.029137348756194115, + 0.03554438799619675, + 0.0018585354555398226, + -0.032696764916181564, + -0.024140356108546257, + -0.021357236430048943, + -0.028801685199141502, + 0.011010331101715565, + 0.021650655195116997, + -0.0010575209744274616, + -0.03997921198606491, + 0.029092639684677124, + -0.028146367520093918, + -0.03686906397342682, + -0.030634339898824692, + 0.02338530123233795, + -0.016822967678308487, + -0.02829514443874359, + 0.04996874928474426, + 0.04095814749598503, + -0.017035555094480515, + 0.031025774776935577, + 0.02043657749891281, + -0.007419919595122337, + 0.009148522280156612, + 0.01298330444842577, + -0.009260712191462517, + -0.014296050183475018, + 0.05299435928463936, + -0.015664828941226006, + 0.009693646803498268, + -0.022403476759791374, + -0.007872791960835457, + 0.037603169679641724, + -0.025187276303768158, + -0.001616489957086742, + 0.0017415047623217106, + 0.007157173939049244, + 0.05363519489765167, + 0.03664301708340645, + 0.03202127665281296, + -0.017482178285717964, + -0.03575655445456505, + -0.011019701138138771, + -0.009754680097103119, + -0.034841228276491165, + -0.022782914340496063, + 0.004245222080498934, + -0.00539123360067606, + 0.004125595558434725, + 0.029694393277168274, + -0.0006681059603579342, + 0.03376293554902077, + -0.02774621546268463, + 0.00667398888617754, + -0.000663229264318943, + 0.022157905623316765, + -0.055641066282987595, + -0.04202336072921753, + 0.0023314678110182285, + -0.009149683639407158, + 0.00029111001640558243, + -0.016934270039200783, + 0.010580064728856087, + -0.01621546410024166, + 0.04160505533218384, + -0.04374832659959793, + -0.027808770537376404, + 0.03193933516740799, + -0.02946849912405014, + 0.03126109018921852, + -0.04800284653902054, + 0.002497292822226882, + -0.011747639626264572, + -0.03316395357251167, + -0.048058319836854935, + -0.026864321902394295, + 0.010974209755659103, + 0.03274235501885414, + 0.02195029892027378, + -0.0364990271627903, + 0.014315557666122913, + 0.027323376387357712, + -0.03487299755215645, + -0.0119673702865839, + 0.0024898534175008535, + 0.024241290986537933, + -0.030937975272536278, + 0.05327417701482773, + -0.03570805490016937, + 0.023580430075526237, + 0.011974279768764973, + 0.009311908856034279, + 0.06938309967517853, + -0.002456553280353546, + -0.020798536017537117, + 0.013593376614153385, + 0.0041114212945103645, + -0.029061079025268555, + 0.04194881394505501, + 0.04250725731253624, + 0.01404556818306446, + 0.024271370843052864, + -0.009714219719171524, + -0.003687768243253231, + 0.041786741465330124, + -0.007013052701950073, + 0.026848822832107544, + 0.05671345070004463, + -0.030590035021305084, + 0.05585445091128349, + 0.014757020398974419, + -0.002260111039504409, + -0.017622731626033783, + 0.027619117870926857, + -0.049994975328445435, + -0.0127108721062541, + -0.05127391964197159, + 0.07030139863491058, + -0.0072503346018493176, + -0.004302551969885826, + 0.026072559878230095, + 0.015258552506566048, + 0.02855512872338295, + 0.01072134543210268, + -0.021203648298978806, + 0.011597119271755219, + -0.0013261126587167382, + -0.0394064225256443, + 0.01728297397494316, + -0.0075342305935919285, + 0.030283741652965546, + -0.031045706942677498, + -0.017537077888846397, + 0.02641989290714264, + -0.0029093874618411064, + -0.03913825750350952, + -0.023756667971611023, + -0.027615567669272423, + 0.0272672101855278, + -0.021244889125227928, + -0.00015110793174244463, + -0.02476469986140728, + 0.02170214056968689, + -0.01185606699436903, + 0.03530268371105194, + 0.028607400134205818, + 0.02599131315946579, + -0.02797403745353222, + 0.0446346253156662, + -0.09909718483686447, + 0.012603332288563251, + -0.011140738613903522, + 0.04496930539608002, + 0.05277857184410095, + 0.0276770181953907, + -0.013171725906431675, + 0.013913131318986416, + -0.01734163425862789, + -0.02198808267712593, + -0.028662677854299545, + 0.012819496914744377, + 0.055279385298490524, + -0.01400172058492899, + 0.0023814004380255938, + -0.012004305608570576, + 0.05265796557068825, + 0.027300450950860977, + -0.015097182244062424, + 0.014795774593949318, + 0.039228759706020355, + -0.02551250532269478, + -0.020109305158257484, + 0.0013544735265895724, + 0.010581343434751034, + -0.0038877837359905243, + 0.001409292919561267, + -0.047337234020233154, + 0.01559543702751398, + -0.036163490265607834, + 0.009957993403077126, + -0.06697048991918564, + -0.03378071263432503, + 0.0011239899322390556, + 0.011381137184798717, + 0.0005068380851298571, + 0.05434541776776314, + -0.022579466924071312, + -0.005730104632675648, + 0.02480519376695156, + -0.024274887517094612, + -0.03358810022473335, + -0.026106132194399834, + 0.03634916990995407, + 0.02683170512318611, + 0.02967071533203125, + 0.04977185279130936, + 0.0035363573115319014, + 0.13878417015075684, + -0.025552846491336823, + 0.027044497430324554, + 0.00979602336883545, + 0.03982779011130333, + -0.018318554386496544, + 0.0034962589852511883, + -0.019270094111561775, + 0.0011834806064143777, + 0.02710220403969288, + -0.027084404602646828, + 0.07396116852760315, + 0.04461243748664856, + 0.04173586890101433, + 0.026143526658415794, + -0.01314216386526823, + 0.01208410132676363, + -0.022636285051703453, + 0.0041359951719641685, + -0.004232829436659813, + 0.014592030085623264, + -0.035695116966962814, + -0.018119139596819878, + -0.006916306912899017, + 0.03456149250268936, + 0.046472176909446716, + 0.020568834617733955, + 0.02278869040310383, + -0.008931105025112629, + 0.007208825089037418, + 0.005493953358381987, + -0.01780230738222599, + 0.07798954099416733, + -0.0003881489101331681, + 0.03840429708361626, + -0.04279085248708725, + 0.006508538965135813, + 0.05804835259914398, + 0.0009788289899006486, + -0.02312953770160675, + 0.035913579165935516, + -0.029228923842310905, + -0.025192376226186752, + 0.024310847744345665, + 0.020029908046126366, + 0.018401185050606728, + -0.005356344860047102, + -0.058178842067718506, + -0.05637163296341896, + 0.008227583020925522, + -0.07202678918838501, + 0.0068654888309538364, + -0.038255538791418076, + 0.0013745547039434314, + -0.04548782482743263, + 0.014020882546901703, + -0.036764755845069885, + -0.049168042838573456, + -0.030310317873954773, + -0.03091461770236492, + -0.008800501935184002, + -0.016487542539834976, + -0.05661512911319733, + -0.030214060097932816, + -0.021353131160140038, + -0.0012679414357990026, + 0.006362128537148237, + -0.027526650577783585, + 0.005728166550397873, + -0.04520412161946297, + 0.039394311606884, + -0.004215404391288757, + 0.021022776141762733, + -0.03609558194875717, + 0.06348548829555511, + -0.032730214297771454, + 0.020463692024350166, + 0.003444208297878504, + -0.027185924351215363, + 0.028515774756669998, + -0.009311286732554436, + 0.046261467039585114, + -0.01657440885901451, + -0.04178300127387047, + 0.03191223368048668, + 0.02114085480570793, + 0.04172463342547417, + 0.018021555617451668, + -0.03130093216896057, + 0.015913013368844986, + 0.011658311821520329, + -0.0019959176424890757, + -0.021104760468006134, + 0.009251799434423447, + 0.00721253827214241, + 0.013789547607302666, + 0.05629168078303337, + 0.00924333743751049, + 0.004671763628721237, + -0.023904908448457718, + -0.015384670346975327, + 0.020198699086904526, + -0.044496797025203705, + 0.01173888798803091, + 0.04518592730164528, + 0.013877086341381073, + 0.0005916933296248317, + 0.009606024250388145, + 0.011999666690826416, + -0.0670611709356308, + -0.030197344720363617, + -0.005791814066469669, + 0.03946763277053833, + 0.019297827035188675, + 0.058342095464468, + -0.00875124055892229, + -0.04651189222931862, + 0.012448364868760109, + -0.053018368780612946, + 0.03512851893901825, + -0.016904424875974655, + -0.014503130689263344, + -0.06344377249479294, + 0.030705086886882782, + 0.010135791264474392, + -0.026598220691084862, + 0.02645527385175228, + -0.015493737533688545, + 0.026033159345388412, + 0.00018316326895728707, + 0.005027357954531908, + -0.040044207125902176, + 0.004479025024920702, + 0.01091387402266264, + -0.041527800261974335, + 0.04933817312121391, + 0.015318014658987522, + -0.03873354569077492, + 0.02157134748995304, + 0.0026436042971909046, + 0.04029274359345436, + -0.09519004076719284, + 0.037977706640958786, + -0.029262162744998932, + -0.008651859126985073, + -0.024707600474357605, + -0.011262014508247375, + -0.015327460132539272, + 0.028289884328842163, + 0.03414604067802429, + 0.05550688877701759, + -0.01389060914516449, + -0.021668963134288788, + -0.012188787572085857, + 0.0012107252841815352, + -0.13569262623786926, + -0.005229906179010868, + -0.011269676499068737, + 0.0071484739892184734, + 0.03034362755715847, + -0.006770261563360691, + -0.001919554197229445, + -0.0008236062130890787, + 0.012352393008768559, + 0.017413541674613953, + -0.03601384162902832, + 0.007097562775015831, + 0.008059581741690636, + 0.04850142076611519, + 0.007137021515518427, + -0.049293648451566696, + -0.05056758597493172, + -0.0010526523692533374, + 0.0030640389304608107, + -0.0352141335606575, + -0.043950922787189484, + -0.008843391202390194, + 0.0759638324379921, + -0.0029227344784885645, + 0.027559084817767143, + 0.021414395421743393, + -0.061646535992622375, + 0.041031938046216965, + -0.04614561051130295, + 0.037430539727211, + -0.043840885162353516, + -0.00905811320990324, + -0.038444943726062775, + -0.006827656179666519, + 0.009538911283016205, + 0.07512883841991425, + 0.004221929237246513, + 0.021550703793764114, + -0.005782400723546743, + 0.046900708228349686, + -0.01129245012998581, + -0.027230193838477135, + -0.04440127685666084, + -0.04526139795780182, + -0.004848231561481953, + -0.045245811343193054, + -0.04045155644416809, + -0.01775304414331913, + 0.04419105872511864, + 0.00899084098637104, + 0.02677333541214466, + 0.016088344156742096, + -0.008500684052705765, + -0.002065642736852169, + -0.02245795913040638, + 0.0004948987625539303, + -0.0035641943104565144, + 0.08004654198884964, + 0.02226107567548752, + -0.019138233736157417, + 0.026443270966410637, + -0.021264702081680298, + 0.017800763249397278, + 0.009022845886647701, + 0.02559858374297619, + -0.02740759216248989, + 0.04680027812719345, + 0.006379961501806974, + 0.0337391160428524, + 0.0076416730880737305, + 0.04804108291864395, + -0.03051508404314518, + -0.013092133216559887, + -0.022701865062117577, + -0.0018052313243970275, + -0.028484933078289032, + 0.005104963667690754, + -0.023172225803136826, + -0.03214229643344879, + 0.015481756068766117, + -0.017329588532447815, + 0.014163349755108356, + -0.007649875711649656, + -0.009236500598490238, + -0.030296672135591507, + -0.009010507725179195, + -0.0405266135931015, + -0.005483418703079224, + 0.00696283346042037, + -0.0018789718160405755, + 0.042392004281282425, + 0.000534349586814642, + -0.013847471214830875, + -0.015025805681943893, + 0.003559636417776346, + -0.008827896788716316, + 0.014901130460202694, + 0.023767098784446716, + 0.025841623544692993, + 0.009565561078488827, + -0.008301333524286747, + -0.027558861300349236, + -0.013756075873970985, + -0.018154241144657135, + -0.02474043518304825, + -0.04755142703652382, + 0.014959920197725296, + -0.07346094399690628, + 0.013546811416745186, + 0.06642317771911621, + 0.04891495779156685, + 0.011619308963418007, + 0.006325960159301758, + 0.013888382352888584, + 0.0016194244381040335, + 0.01198501791805029, + -0.004664474632591009, + 0.02830151468515396, + -0.044690169394016266, + 0.02557440660893917, + -0.020120937377214432, + -0.041543908417224884, + 0.024267133325338364, + -0.0003947774530388415, + 0.0008917154627852142, + -0.004224342294037342, + -0.0050197322852909565, + 0.007586198393255472, + -0.018146267160773277, + 0.058293331414461136, + 0.05371890589594841, + -0.0034453365951776505, + -0.010232423432171345, + -0.04612710326910019, + -0.002813778119161725, + -0.005883947480469942, + -0.0639842227101326, + -0.05570071190595627, + 0.037858057767152786, + -0.029446180909872055, + -0.042994238436222076, + 0.006562717258930206, + 0.08181930333375931, + -0.027989549562335014, + 0.013632590882480145, + 0.02700175903737545, + 0.04160214588046074, + 0.003348167287185788, + -0.00543568842113018, + -0.010110131464898586, + -0.005173802375793457, + 0.006112094037234783, + -0.047498323023319244, + -0.011902482248842716, + 0.028048919513821602, + 0.010043435730040073, + 0.03247036412358284, + -0.006244494114071131, + -0.06105179712176323, + 0.03210403025150299, + 0.01459363754838705, + 0.016195455566048622, + -0.0177913811057806, + 0.03381579369306564, + 0.04794391244649887, + 0.0023042054381221533, + -0.054043419659137726, + -0.029858019202947617, + -0.030908819288015366, + -0.02212235890328884, + 0.03805728629231453, + 0.05044630914926529, + 0.02300877682864666, + 0.0005204136832617223, + -0.044941551983356476, + -0.024526042863726616, + 0.03855337202548981, + -0.03974326327443123, + -0.005814287345856428, + -0.039317537099123, + -0.023211613297462463, + -0.04497330263257027, + 0.02936973050236702, + -0.00639959704130888, + -0.004922559484839439, + -0.01788298785686493, + -0.03759142756462097, + 0.0437617152929306, + -0.014056861400604248, + 0.012868726626038551, + -0.007818229496479034, + 0.01195360254496336, + 0.012582234106957912, + 0.04740920662879944, + -0.059802137315273285, + 0.02556523308157921, + -0.02762327715754509, + 0.007077451795339584, + 0.01842229627072811, + -0.03334515541791916, + 0.002066141227260232, + 0.004608731251209974, + 0.03950078412890434, + -0.021999355405569077, + 0.040115389972925186, + 0.03122766502201557, + -0.0019269372569397092, + -0.016822321340441704, + 0.052588801831007004, + 0.030341848731040955, + -0.010891610756516457, + -0.009114057756960392, + -0.013973744586110115, + 0.01381004136055708, + -0.01438795868307352, + 0.00990266539156437, + 0.02910882793366909, + 0.1272100806236267, + 0.02174985408782959, + 0.041390560567379, + 0.03742929548025131, + -0.022338788956403732, + 0.015667829662561417, + 0.030061742290854454, + 0.09463869780302048, + -0.01416815910488367, + -0.03746543824672699, + -0.002283993177115917, + -0.022235065698623657, + 0.039771176874637604, + 0.056245092302560806, + -0.03456134349107742, + 0.01892317831516266, + -0.017168859019875526, + 0.027797944843769073, + -0.06501670181751251, + -0.02500161901116371, + 0.006215489469468594, + -0.0614696741104126, + -0.021722596138715744, + -0.019558805972337723, + -0.020642191171646118, + -0.05282612144947052, + -0.046975117176771164, + -0.028187628835439682, + 0.034627169370651245, + -0.000985091901384294, + -0.02344481460750103, + 0.014718118123710155, + -0.054312583059072495, + 0.02367238886654377, + 0.04747230187058449, + -0.01497164648026228, + -0.0032743148040026426, + -0.013978242874145508, + 0.03980674222111702, + -0.03044397570192814, + 0.053500350564718246, + 0.01707790605723858, + 0.02395015023648739, + -0.0249787587672472, + -0.0045135668478906155, + -0.0075399000197649, + 0.03243964910507202, + 0.04173853248357773, + 0.017002267763018608, + -0.001506801345385611, + 0.004195653833448887, + 0.024452058598399162, + -0.050133250653743744, + 0.006282606162130833, + -0.012643913738429546, + -0.013555298559367657, + 0.02532607689499855, + -0.024098213762044907, + -0.040130700916051865, + -0.0159552413970232, + 0.025651585310697556, + -0.01976807415485382, + -0.01051683072000742, + -0.026881923899054527, + 0.020509611815214157, + 0.0373283252120018, + 0.011705932207405567, + 0.00615881010890007, + 0.025379763916134834, + 0.018891464918851852, + 0.0024790673051029444, + -0.01691329851746559, + -0.046437300741672516, + 0.0038868391420692205, + -0.0470905639231205, + 0.004895806312561035, + 0.03212757036089897, + -0.0033503028098493814, + 0.01075777504593134, + -0.03628802299499512, + -0.009073710069060326, + -0.013223791494965553, + -0.031178459525108337, + -0.03752364218235016, + 0.01817743107676506, + 0.01719512790441513, + -0.033583443611860275, + -0.04240895062685013, + -0.00770330335944891, + -0.006301648449152708, + -0.019367823377251625, + -0.0015334236668422818, + -0.029566805809736252, + -0.01342070009559393, + 0.02468663454055786, + -0.013153699226677418, + 0.03467167541384697, + -0.019284095615148544, + -0.009039157070219517, + 0.15529383718967438, + 0.0034466793294996023, + -0.004056619480252266, + 0.011022305116057396, + -0.013875667005777359, + 0.04483966901898384, + 0.019348017871379852, + 0.04122842103242874, + 0.026633935049176216, + 0.024539906531572342, + 0.04840689152479172, + -0.04734452813863754, + -0.00993296317756176, + -0.049822766333818436, + -0.04119617119431496, + -0.06432229280471802, + 0.027639027684926987, + 0.005802113562822342, + 0.022075871005654335, + 0.005319235380738974, + -0.03515484556555748, + -0.0005135624087415636, + -0.03402785584330559, + 0.07831410318613052, + 0.02364080585539341, + -0.009090916253626347, + -0.05810858681797981, + -0.018819885328412056, + -0.03755798190832138, + -0.015169009566307068, + 0.002187391510233283, + -0.01697169803082943, + 0.0013278487604111433, + 0.049949511885643005, + 0.022530682384967804, + -0.018425263464450836, + 0.004223344847559929, + 0.0025568241253495216, + 0.03026408515870571, + 0.03267018869519234, + -0.03235092759132385, + 0.03720671683549881, + 0.00677361199632287, + 0.04721737653017044, + 0.03617458418011665, + 0.04278988763689995, + -0.019048823043704033, + 0.010050734505057335, + 0.00972666684538126, + 0.012449121102690697, + -0.005297062918543816, + -0.005260234698653221, + 0.010130810551345348, + 0.0017256037099286914, + -0.0215604230761528, + -0.06170865148305893, + 0.00280454708263278, + -0.005960467271506786, + 0.020882336422801018, + 0.020411213859915733, + -0.05604246258735657, + 0.011033855378627777, + -0.010959495790302753, + -0.01250424887984991, + 0.016010824590921402, + 0.02409837581217289, + 0.04092054069042206, + 0.017150159925222397, + -0.017616592347621918, + 0.07128634303808212, + 0.02309083752334118, + -0.011149837635457516, + -0.02061529830098152, + 0.004348506685346365, + -2.2379419533535838e-05, + 0.040002111345529556, + 0.07259580492973328, + 0.01128822099417448, + -0.01100581232458353, + 0.005588391795754433, + 0.024579152464866638, + -0.014415424317121506, + 0.007610009051859379, + 0.0220401119440794, + 0.04975166916847229, + 0.051722291857004166, + -0.03055739775300026, + -0.0066607934422791, + 0.004166224040091038, + -0.01772529073059559, + -0.024240216240286827, + -0.03705402463674545, + -0.025252236053347588, + 0.01835777796804905, + -0.0021136831492185593, + -0.023405106738209724, + 0.006622226908802986, + -0.03978988155722618, + 0.025801122188568115, + -0.013518063351511955, + 0.006525570992380381, + 0.025944795459508896, + 0.02185557596385479, + 0.009526408277451992, + 0.05153016373515129, + 0.0160360224545002, + -0.01775054819881916, + -0.025139443576335907, + -0.027722341939806938, + -0.062106069177389145, + 0.00649713771417737, + -0.022500593215227127, + 0.018269119784235954, + 0.03990677371621132, + -0.0557236447930336, + 0.016872549429535866, + 0.019718484953045845, + 0.008431028574705124, + 0.0029825838282704353, + 0.0385812409222126, + 0.09513341635465622, + -0.020673047751188278, + 0.0755668580532074, + -0.01921212486922741, + -0.026464730501174927, + 0.041091665625572205, + 0.03609565645456314, + -0.006259865127503872, + -0.004000209271907806, + 0.004330358933657408, + 0.013215664774179459, + 0.02991190180182457, + -0.043726034462451935, + 0.02545894682407379, + 0.030375070869922638, + -0.022892950102686882, + -0.02041018009185791, + -0.07082711905241013, + -0.04964400827884674, + 0.0088781313970685, + 0.010395539924502373, + 0.03429747000336647, + -0.020109759643673897, + -0.013640628196299076, + 0.03482900187373161, + -0.02911890298128128, + -0.0174860879778862, + -0.00202310923486948, + 0.0012273018946871161, + 0.03561239317059517, + 0.012942701578140259, + -0.024896563962101936, + 0.022942857816815376, + 0.002280533080920577, + 0.0351436510682106, + -0.0019736632239073515, + 0.011795478872954845, + -0.01606215350329876, + -0.05100564286112785, + 0.012963490560650826, + 0.048263903707265854, + -0.03669837489724159, + 0.001692362129688263, + -0.0286725964397192, + -0.01656062714755535, + 0.03961143642663956, + -0.003952363505959511, + 0.013821360655128956, + -0.0023555876687169075, + 0.005498651415109634, + 0.008495034649968147, + -0.037946056574583054, + -0.05308791995048523, + 0.004253003746271133, + -0.052943017333745956, + 0.04678841680288315, + -0.03709978610277176, + -0.026549067348241806, + 0.010027294047176838, + -0.045030441135168076, + 0.034882061183452606, + -0.008430280722677708, + -0.02592455968260765, + 0.0627257376909256, + -0.02478695660829544, + -0.0610731802880764, + -0.07562141865491867, + -0.012200155295431614, + 0.013554604724049568, + -0.08892181515693665, + 0.012729254551231861, + 0.021676069125533104, + -0.011317877098917961, + 0.009110948070883751, + -0.016120556741952896, + 0.011451172642409801, + 0.021223654970526695, + -0.002229081466794014, + 0.0032122705597430468, + -0.005442687310278416, + -0.011567797511816025, + -0.025818444788455963, + 0.011701194569468498, + -0.008201606571674347, + 0.04293695464730263, + 0.03949956223368645, + -0.012551640160381794, + 0.009162753820419312, + -0.022905582562088966, + -0.04056906700134277, + 0.01445603184401989, + -0.03786734864115715, + -0.010307714343070984, + -0.03652647137641907, + 0.03070850484073162, + -0.02887650765478611, + 0.0017429434228688478, + 0.03116706945002079, + -0.021935025230050087, + 0.016734091565012932, + -0.04388038441538811, + 0.008103277534246445, + 0.003308236366137862, + 0.02441919595003128, + -0.007449350319802761, + 0.03731127828359604, + -0.01989126019179821, + 0.031498022377491, + -0.02314888685941696, + -0.02022203616797924, + -0.03748361021280289, + 0.07238303869962692, + -9.908675565384328e-05, + 0.0023647258058190346, + -0.02368614822626114, + -0.01090051606297493, + 0.030961494892835617, + -0.017382360994815826, + 0.005312441382557154, + -0.01257698517292738, + -0.03836457431316376, + 0.014171610586345196, + -0.03415873274207115, + -0.023835115134716034, + 0.021407941356301308, + -0.0006461238372139633, + -0.02128068543970585, + -0.03326443210244179, + 0.023638134822249413, + 0.030378680676221848, + -0.012899587862193584, + -0.004305344074964523, + -0.008569394238293171, + -0.05255923047661781, + 0.06379973888397217, + -0.02424098178744316, + 0.023325474932789803, + -0.04297333583235741, + -0.003917680121958256, + 0.009505841881036758, + 0.0010013502324, + 0.016960209235548973, + -0.011692003346979618, + -0.023668423295021057, + -0.0023925211280584335, + -0.03395519405603409, + 0.04240674525499344, + 0.005047050770372152, + 0.05130983889102936, + 0.009539725258946419, + -0.02674953266978264, + -0.012807551771402359, + 0.017234649509191513, + 0.018329212442040443, + 0.004895556252449751, + 0.01752137951552868, + 7.785223715472966e-06, + -0.024250280112028122, + -0.049110934138298035, + -0.022145193070173264, + 0.03123808279633522, + -0.016290059313178062, + 0.04273059591650963, + -0.025241466239094734, + -0.04488757625222206, + 0.01004115305840969, + -0.0034456513822078705, + -0.05110643059015274, + 0.008796930313110352, + 0.012167827226221561, + 0.013582397252321243, + 0.004804785363376141, + -0.032778218388557434, + 0.053487204015254974, + -0.021581323817372322, + 0.009105950593948364, + 0.005368262063711882, + 0.001473954296670854, + -0.01318971998989582, + -0.02805495075881481, + 0.05488653853535652, + 0.045631542801856995, + -0.004000747110694647, + -0.02070310153067112, + -0.02394949086010456, + 0.05110957846045494, + -0.04092952609062195, + -0.026369381695985794, + -0.02583877183496952, + -0.0014783891383558512, + 0.03094647452235222, + 0.0019732178188860416, + -0.012526133097708225, + -0.006310418713837862, + 0.026956532150506973, + 0.015997624024748802, + 0.008467467501759529, + -0.02444959431886673, + 0.006471547763794661, + -0.04544704034924507, + -0.014686410315334797, + -0.017112845554947853, + -0.08452935516834259, + -0.004182206001132727, + -0.011247392743825912, + -0.0035992651246488094, + 0.02940000221133232, + -0.03593040630221367, + 0.010390127077698708, + 0.0015767785953357816, + 0.040778808295726776, + 0.035640690475702286, + 0.029059998691082, + -0.04834333807229996, + 0.015629520639777184 + ], + "attn_peak_rel": -2.25, + "attn_entropy": 0.13674996281042695, + "attn_spread_pm2": 0.5153277148492634, + "attn_role": "looks_back", + "n_pairs": 8 + }, + "CONDITION": { + "l_star": 19, + "h_dom": 14, + "snr": 1.3535798114174444, + "specificity": 1.6998493038806486, + "canonical_vec": [ + 0.010645583271980286, + 0.02672751061618328, + 0.0036646511871367693, + -0.018149249255657196, + -0.023868359625339508, + -0.008992336690425873, + -0.008582649752497673, + -0.005640093237161636, + -0.000598395592533052, + 0.2040536105632782, + 0.024494361132383347, + 0.02128203958272934, + 0.026714852079749107, + 0.020794203504920006, + 0.004623171407729387, + -0.00995326042175293, + -0.006599421612918377, + 0.006201403681188822, + 0.015498686581850052, + -0.0021972062531858683, + 0.0018743505934253335, + 0.01845238171517849, + 0.0391930416226387, + 0.03382963687181473, + 0.01664193719625473, + -0.0012347042793408036, + 0.03204050287604332, + 0.0010467732790857553, + 0.006095800083130598, + -0.0011759580811485648, + 0.013621575199067593, + 0.028939243406057358, + -0.03751472756266594, + 0.008083174005150795, + -0.023840736597776413, + 0.009656685404479504, + -0.021066805347800255, + 0.00175005744677037, + -0.004547106102108955, + 0.027417410165071487, + -0.007217081729322672, + 0.020572174340486526, + -0.03348604589700699, + -0.010943328961730003, + 0.0061827716417610645, + -0.0065983436070382595, + -0.028098607435822487, + 0.00589690450578928, + -0.006025348324328661, + 0.01620114967226982, + 0.03254147619009018, + 0.05098355561494827, + 0.0063279555179178715, + 0.0008206660859286785, + -0.007060503587126732, + 0.020693622529506683, + -0.014673086814582348, + 0.01796303503215313, + 0.002215753309428692, + 0.009276030585169792, + -0.012875518761575222, + -0.009900659322738647, + 0.044639717787504196, + -0.013605914078652859, + 0.005423972848802805, + 0.006641997490078211, + 0.008243454620242119, + 0.00559771666303277, + -0.015949588268995285, + -0.020264875143766403, + 0.026914939284324646, + -0.002710906323045492, + 0.007815550081431866, + 0.019923217594623566, + -0.041588377207517624, + -0.013884235173463821, + 0.03414027392864227, + -0.003547713393345475, + 0.028728879988193512, + 0.00823912862688303, + -0.010772747918963432, + 0.02557372860610485, + 0.048843320459127426, + 0.03701190650463104, + -0.015254094265401363, + -0.029855534434318542, + 0.04050300642848015, + -0.005353073123842478, + 0.005637017078697681, + -0.001794637180864811, + 0.008200053125619888, + 0.015046624466776848, + 0.0009710132144391537, + -0.010064935311675072, + -0.0007378552691079676, + 0.03788207843899727, + 0.038903024047613144, + -0.015168054960668087, + 0.024322252720594406, + 0.0020949975587427616, + 0.0017170312348753214, + 0.0037805817555636168, + 0.005536214914172888, + -0.004855453036725521, + 0.011646064929664135, + -0.035526640713214874, + -0.004288434982299805, + 0.022907374426722527, + -0.011147018522024155, + -0.009950809180736542, + -0.0011946483282372355, + -0.02544581890106201, + -0.04171218350529671, + -0.016702566295862198, + -0.014947592280805111, + -0.006826482713222504, + 0.017331568524241447, + 0.006677576340734959, + 0.006287813186645508, + 0.03424360975623131, + -0.029407434165477753, + 0.015512948855757713, + 0.007291705347597599, + -0.0028714225627481937, + -0.029279662296175957, + 0.001731813419610262, + -0.01241394504904747, + 0.008581455796957016, + 0.009523062035441399, + -0.07458668202161789, + -0.015244217589497566, + 0.03558184579014778, + 0.01424409169703722, + -0.018842602148652077, + 0.03659245744347572, + -0.0027998890727758408, + 0.020358620211482048, + -0.002204709453508258, + 0.012970788404345512, + -0.03199757635593414, + -0.011226154863834381, + 0.012419614940881729, + -0.017972877249121666, + -0.04247841238975525, + 0.018949229270219803, + -0.02958480827510357, + 0.0008280486799776554, + 0.044100891798734665, + 0.006145155057311058, + 0.03164707496762276, + 0.03237464651465416, + -0.024773990735411644, + 0.004896020516753197, + 0.015011734329164028, + 0.026154013350605965, + 0.07031839340925217, + -0.0008208296494558454, + 0.03834497556090355, + 0.03165234252810478, + 0.007221083622425795, + 0.01486407034099102, + 0.03953694924712181, + -0.017356006428599358, + 0.007300146855413914, + 0.020704610273241997, + 0.031006451696157455, + -0.011829420924186707, + -0.0030834113713353872, + -0.0020935216452926397, + -0.01750863529741764, + -0.021603945642709732, + 0.008233367465436459, + 0.009324107319116592, + 0.02146257646381855, + 0.025528373196721077, + 0.013432363979518414, + 0.010217029601335526, + 0.012941475957632065, + 0.016360068693757057, + 0.0062989890575408936, + -0.008512790314853191, + -0.00045089860213920474, + 0.01272672787308693, + 0.0077886092476546764, + -0.08006345480680466, + -0.010732141323387623, + 0.008070806041359901, + 0.022334016859531403, + 0.02117169462144375, + -0.017102546989917755, + -0.019606931135058403, + 0.005302281118929386, + 0.005874922964721918, + -0.02028348110616207, + -0.020773690193891525, + -0.016657629981637, + -0.03293927013874054, + 0.001342478091828525, + 0.016771545633673668, + 0.0009426656179130077, + -0.012775647453963757, + -0.03632676228880882, + -0.010232020169496536, + -0.026584891602396965, + 0.016537267714738846, + -0.046579401940107346, + -0.0060998848639428616, + 0.0341319739818573, + 0.02732340805232525, + -0.016475481912493706, + -0.0026579361874610186, + 0.013678114861249924, + -0.021993448957800865, + -0.038828667253255844, + -0.05024847760796547, + -0.014171585440635681, + 0.006240123882889748, + 0.03711142763495445, + 0.00430071959272027, + 0.054979357868433, + 0.0017049796879291534, + -0.004096776247024536, + 0.020996084436774254, + -0.015759512782096863, + 0.02357623167335987, + 0.04208037629723549, + 0.01913835108280182, + 0.023598967120051384, + -0.018354909494519234, + 0.016694270074367523, + 0.012293563224375248, + -0.07200358062982559, + 0.0006959835300222039, + -0.006614739540964365, + 0.020218463614583015, + -0.019912371411919594, + -0.030936188995838165, + -0.005859701428562403, + -0.012709853239357471, + -0.032384246587753296, + -0.01949113793671131, + -0.011239124462008476, + -0.003917926922440529, + 0.01945808343589306, + -0.006861046887934208, + -0.006182681303471327, + -0.007444858085364103, + -0.008268498815596104, + -0.019638625904917717, + -0.014117675833404064, + -0.009540392085909843, + 0.003893611952662468, + 0.013840019702911377, + -0.008569690398871899, + -0.021868733689188957, + -0.010285216383635998, + -0.030600523576140404, + 0.000135296635562554, + -0.02047419734299183, + -0.034939493983983994, + -0.02081364393234253, + -0.015847649425268173, + -0.0015531901735812426, + -0.013919055461883545, + -0.023355966433882713, + 0.038099441677331924, + 0.02398855984210968, + -0.011172441765666008, + 0.6549933552742004, + 0.005037899594753981, + 0.027004770934581757, + -0.010341563262045383, + -0.012698134407401085, + 0.03204628825187683, + 0.019938483834266663, + -0.00353593030013144, + -0.01900821179151535, + -0.001118809450417757, + 0.04687387868762016, + 0.015301446430385113, + 0.048207543790340424, + 0.01019976194947958, + -0.006527194753289223, + 0.009048775769770145, + -0.011976205743849277, + 0.016953835263848305, + 0.030420800670981407, + -0.011287948116660118, + -0.009773816913366318, + -0.01756516844034195, + 0.002694101305678487, + 0.007268215529620647, + -0.012324675917625427, + -0.0034036505967378616, + -0.028491662815213203, + -0.0104835731908679, + 0.010076072998344898, + 0.025307882577180862, + 0.04379311949014664, + -0.02214166149497032, + -0.006540048401802778, + -0.003284753067418933, + 0.005339490715414286, + -0.0033918838016688824, + 0.018712308257818222, + -0.018844109028577805, + -0.008505537174642086, + 0.03394979611039162, + -0.019220726564526558, + 0.006150887813419104, + 0.008438250981271267, + -0.027964720502495766, + 0.053418319672346115, + -0.008439162746071815, + -0.02313804440200329, + 0.0070555806159973145, + -0.014992939308285713, + 0.004279629793018103, + 0.048522818833589554, + -0.0031538933981209993, + -0.0037488609086722136, + -0.010266369208693504, + -0.023744797334074974, + 0.0049184090457856655, + -0.015740230679512024, + 0.021714936941862106, + 0.011668029241263866, + -0.02194465883076191, + -0.0226912721991539, + -0.010263573378324509, + 0.020207902416586876, + -0.028492461889982224, + 0.01751883700489998, + -0.010386920534074306, + -0.030018191784620285, + 0.010709568858146667, + 0.01053659524768591, + -0.02336302027106285, + 0.03991205617785454, + -0.02000775747001171, + 0.0058050756342709064, + 0.03622914105653763, + 0.00020031021267641336, + 0.0042905062437057495, + -0.030591001734137535, + -0.05801470950245857, + -0.0430854931473732, + 0.012110378593206406, + 0.023413419723510742, + -0.007905895821750164, + -0.023431330919265747, + 0.027133295312523842, + 0.03363293781876564, + -0.018998263403773308, + -0.009028024971485138, + 0.02903163805603981, + -0.011736004613339901, + -0.002433828543871641, + -0.025508752092719078, + -0.006981561426073313, + -0.036725159734487534, + -0.00033318501664325595, + 0.005846288520842791, + -0.022626614198088646, + 0.05155982822179794, + -0.02165292017161846, + -0.009475840255618095, + 0.04148111119866371, + -0.012695403769612312, + -0.028323402628302574, + -0.00046032085083425045, + -0.016733454540371895, + -0.03933989629149437, + 0.00977061502635479, + -0.010683946311473846, + -0.009204051457345486, + 0.057193823158741, + -0.007163005881011486, + 0.01125738862901926, + 0.0031123515218496323, + 0.0022922023199498653, + -0.011276374571025372, + 0.023210516199469566, + 0.018137993291020393, + 0.0086853401735425, + -0.012558164075016975, + 0.013762974180281162, + 0.015028932131826878, + 0.02007356658577919, + -0.009336451068520546, + 0.015887551009655, + 0.01160526555031538, + 0.0021984498016536236, + -0.010347362607717514, + -0.02803528867661953, + 0.026552092283964157, + 0.029940281063318253, + -0.03739314153790474, + 0.014266175217926502, + 0.006801275070756674, + -0.015868838876485825, + 0.015308803878724575, + 0.005105628632009029, + -0.02652440033853054, + 0.00020806990505661815, + -0.03077811934053898, + -0.0014124654699116945, + 0.0029340360779315233, + -0.014250148087739944, + 0.04396625980734825, + -0.003515782533213496, + 0.02778628095984459, + -0.006907314993441105, + -0.017866283655166626, + 0.0010786529164761305, + -0.004224757198244333, + 0.012289363890886307, + -0.01948746293783188, + 0.014015366323292255, + -0.004937955643981695, + 0.012009239755570889, + -0.019128253683447838, + 0.02617359533905983, + 0.020304985344409943, + 0.04125508293509483, + 0.004692415706813335, + 0.005802858155220747, + -0.030486376956105232, + -0.02112368308007717, + 0.014644616283476353, + -0.030483826994895935, + 0.001895539346151054, + 0.019857672974467278, + -0.012345920316874981, + 0.06196130812168121, + -0.03507569804787636, + -0.019170725718140602, + -0.004996034782379866, + 0.021653220057487488, + -0.03310756757855415, + -0.03133254870772362, + -0.010726705193519592, + -0.01977837271988392, + 0.014655186794698238, + -0.042961154133081436, + 0.005617686081677675, + 0.02572893723845482, + -0.03148142248392105, + -0.03172232583165169, + -0.014841063879430294, + 0.005568737164139748, + -0.029018584638834, + 0.034451909363269806, + 0.032535430043935776, + -0.013439830392599106, + 0.024732166901230812, + -0.03256189078092575, + -0.004173383582383394, + 0.014337431639432907, + -0.03935893625020981, + 0.006069190334528685, + 0.014548621140420437, + 0.0012069278163835406, + 0.025789793580770493, + -0.014471354894340038, + 0.0267481692135334, + 0.02784968540072441, + -0.020300110802054405, + -0.011700086295604706, + -0.01250573992729187, + -0.036165907979011536, + 0.007062725722789764, + 0.03662097454071045, + 0.004837378393858671, + 0.030598655343055725, + 0.007393213454633951, + 0.03057068958878517, + -0.014528478495776653, + 0.026966523379087448, + 0.049967311322689056, + -0.021202517673373222, + -0.010976653546094894, + -0.011244691908359528, + -0.008991587907075882, + 0.02516818232834339, + -0.017042944207787514, + -0.04571283608675003, + -0.015863176435232162, + 0.02592960000038147, + 0.010572031140327454, + -0.019559510052204132, + -0.02328258752822876, + 0.014584490098059177, + -0.052975594997406006, + 0.0007071615546010435, + 0.022504214197397232, + -0.07178547978401184, + -0.004231073893606663, + -0.031475212424993515, + 0.007587582338601351, + -0.03998948633670807, + 0.0325399786233902, + -0.05269862711429596, + -0.004282635170966387, + -0.024794286116957664, + 0.030400216579437256, + 0.04571567848324776, + 0.012797984294593334, + -0.0041912090964615345, + 0.014263466000556946, + -0.01859137788414955, + -0.000389322463888675, + 0.024892179295420647, + -0.022425780072808266, + -0.01039811223745346, + -0.018961424008011818, + -0.003461746033281088, + -0.029799988493323326, + -0.023923825472593307, + 0.014695093035697937, + -0.003994347993284464, + 0.007735447958111763, + -0.017635080963373184, + -0.004540823865681887, + -0.01591176725924015, + 0.015011981129646301, + 0.01731298677623272, + 0.025403672829270363, + -0.02108106203377247, + -0.004600054118782282, + -0.019739339128136635, + 0.0031966723036020994, + 0.01820053718984127, + -0.00828089751303196, + 0.005156379193067551, + 0.006582499481737614, + -0.0035397799219936132, + 0.01374864112585783, + -0.016848942264914513, + 0.036358341574668884, + 0.020204413682222366, + -0.006246849428862333, + 0.002459866926074028, + -0.012629633769392967, + 0.0070713721215724945, + -0.00915362499654293, + 0.01579056680202484, + -0.00025056040612980723, + -0.006073815282434225, + 0.01392765250056982, + 0.0032037198543548584, + 0.009923843666911125, + 0.011698496527969837, + 0.02780858427286148, + -0.013815542683005333, + 0.020276887342333794, + -0.009456491097807884, + 0.021823810413479805, + -0.027249248698353767, + -0.028884461149573326, + 0.015015490353107452, + 0.015478957444429398, + -0.020407957956194878, + -0.01599232293665409, + -0.007111475337296724, + -0.02442907728254795, + 0.012905892916023731, + 0.004658674355596304, + 0.009468908421695232, + -0.007659710478037596, + 0.012723951600492, + -0.028595343232154846, + 0.0013020280748605728, + -0.015772663056850433, + -0.010086869820952415, + -0.010944502428174019, + -0.03734654188156128, + 0.0020282920449972153, + -0.02356419712305069, + -0.02554483711719513, + 0.008997957222163677, + 0.03556423634290695, + -0.01669350452721119, + -0.009854494594037533, + -0.009525259956717491, + 0.032107364386320114, + -0.030828261747956276, + -0.020381029695272446, + 0.03445062041282654, + -0.031118454411625862, + -0.0941360741853714, + 0.016317490488290787, + 0.02021845430135727, + 0.008118263445794582, + 0.037053875625133514, + 0.008181513287127018, + 0.007483420893549919, + 0.026788046583533287, + -0.03471182659268379, + -0.015680313110351562, + 0.005815040785819292, + 0.013379666954278946, + -0.00391447264701128, + 0.002699978882446885, + 0.010840443894267082, + 0.015145630575716496, + 0.01685182750225067, + 0.011554909870028496, + -0.02225731685757637, + -0.01790633425116539, + 0.017207209020853043, + -0.022164704278111458, + 0.008928761817514896, + -0.023655544966459274, + 0.005917813628911972, + 0.08714541792869568, + 8.463145786663517e-05, + 0.008113153278827667, + -0.01754954643547535, + 0.005373796448111534, + 0.02322266809642315, + -0.027022257447242737, + 0.008797585032880306, + 0.00011914184869965538, + 0.008142596110701561, + -0.015800001099705696, + 0.01711905561387539, + -0.008022842928767204, + 0.01550557091832161, + -0.011743497103452682, + 0.027872595936059952, + 0.013908438384532928, + 0.024970747530460358, + -0.040444519370794296, + 0.00030436317319981754, + 0.013367093168199062, + 0.007120398338884115, + -0.00861498061567545, + 0.0204768069088459, + -0.041431933641433716, + -0.0393776036798954, + -0.013171597383916378, + -0.0027979419101029634, + -0.0048385807313025, + 0.009649361483752728, + 0.0038376308511942625, + -0.008023724891245365, + 0.005291110835969448, + -0.016599174588918686, + 0.002589994575828314, + 0.024581465870141983, + 0.0468115508556366, + -0.021989304572343826, + -0.017375390976667404, + 0.009349124506115913, + -0.01962602138519287, + 0.009632924571633339, + 0.03355107456445694, + -0.006970931775867939, + -0.03585273399949074, + -0.019307594746351242, + 0.009821411222219467, + -0.0026681709568947554, + -0.05179089307785034, + 0.005505144130438566, + -0.027159012854099274, + -0.0055613890290260315, + -0.022011680528521538, + -0.0009604482911527157, + 0.008979788981378078, + 0.010984350927174091, + -0.08974534273147583, + -0.005945355631411076, + -0.010320482775568962, + -0.025840865448117256, + -0.017289655283093452, + -0.009480947628617287, + -0.024355536326766014, + 0.0011798463528975844, + -0.009585180319845676, + -0.008558501489460468, + 0.01177236158400774, + 0.010864252224564552, + -0.0233325082808733, + 0.010177993215620518, + -0.0007735668332315981, + -0.025321271270513535, + -0.0058413599617779255, + 0.014064916409552097, + -0.043693192303180695, + 0.005668269470334053, + 0.03682669252157211, + 0.025022754445672035, + -0.004323594272136688, + -0.03948434069752693, + 0.025796299800276756, + 6.6930333559867e-05, + 0.041397854685783386, + -0.04237821325659752, + 0.026510288938879967, + -0.002167405327782035, + -0.03305370360612869, + -0.03314246982336044, + 0.0006588852847926319, + -0.0037372747901827097, + 0.04249550774693489, + -0.022995099425315857, + 0.010960966348648071, + -0.022306891158223152, + -0.020879920572042465, + 0.018824158236384392, + 0.0354362428188324, + -0.008389489725232124, + -0.020085949450731277, + -0.0037773700896650553, + 0.011356227099895477, + -0.014559969305992126, + 0.025224387645721436, + -0.025604156777262688, + -0.02401248924434185, + 0.0012620800407603383, + -0.009392726235091686, + -0.0015909551875665784, + 0.00024354936613235623, + 0.01187481265515089, + -0.005933444947004318, + -0.04974600300192833, + 0.022883988916873932, + 0.04310524836182594, + -0.001831845729611814, + 0.02632441744208336, + 0.017426112666726112, + 0.017916034907102585, + -0.0223410502076149, + 0.04326348751783371, + -0.02076917141675949, + 0.011438906192779541, + -0.02860821969807148, + 0.015085762366652489, + -0.022944744676351547, + 0.019571753218770027, + 0.022600801661610603, + -0.014611688442528248, + -0.013233398087322712, + -0.018596479669213295, + 0.0374857597053051, + 0.005370894446969032, + -0.018876954913139343, + -0.018512556329369545, + -0.003198900492861867, + 0.006179729476571083, + -0.01628066785633564, + -0.0319502018392086, + 0.02944250963628292, + -0.0018296984490007162, + -0.0250642579048872, + 0.02813870646059513, + -0.01577177830040455, + 0.018021557480096817, + -0.017376672476530075, + -0.026797650381922722, + 0.008907405659556389, + -0.005228238645941019, + -0.0020027521532028913, + 0.007105253636837006, + -0.011814022436738014, + -0.004826060030609369, + -0.020093834027647972, + 0.016903046518564224, + -0.02584892511367798, + 0.014114056713879108, + 0.014682352542877197, + -0.034268129616975784, + 0.0051361205987632275, + 0.009696959517896175, + 0.06186630204319954, + 0.04238133877515793, + -0.02986101247370243, + -0.0162191204726696, + 0.005655422806739807, + 0.045392923057079315, + -0.043021123856306076, + -0.008296347223222256, + -0.0071665626019239426, + 0.018303778022527695, + 0.0025685392320156097, + -0.020303592085838318, + -0.0022431223187595606, + 0.010032822377979755, + -0.011652512475848198, + -0.015046886168420315, + 0.02274724468588829, + 0.021225251257419586, + -0.04061185196042061, + 0.03224524110555649, + 0.033375680446624756, + 0.04328155145049095, + 0.017886346206068993, + 0.04043091833591461, + 0.02806587889790535, + 0.026736630126833916, + -0.002791705774143338, + 0.0049990322440862656, + -0.015085641294717789, + -0.0429663248360157, + 0.017078200355172157, + -0.004059139173477888, + -0.009861955419182777, + 0.005680770147591829, + -0.01150063332170248, + 0.059479985386133194, + -0.019655998796224594, + 0.04495398700237274, + 0.00682079466059804, + -0.02218790166079998, + 0.010639842599630356, + -0.0352514423429966, + -0.030024707317352295, + 0.00041972153121605515, + 0.033189207315444946, + 0.027637582272291183, + 0.02214319072663784, + 0.005579953547567129, + -0.01712258718907833, + 0.0042961835861206055, + 0.005974657833576202, + 0.02371322177350521, + -0.006423513870686293, + -0.015188478864729404, + -0.0037095320876687765, + -0.0200774185359478, + 0.0532267689704895, + 0.029314782470464706, + 0.045218344777822495, + -0.019614173099398613, + -0.0063743735663592815, + -0.030347995460033417, + 0.007004836108535528, + -0.009668886661529541, + -0.026749802753329277, + 0.022901635617017746, + 0.03695381432771683, + 0.015002347528934479, + -0.03783032298088074, + 0.0012984579661861062, + -0.023957477882504463, + 0.0012613640865311027, + -0.010357537306845188, + -0.019930509850382805, + -0.005657514091581106, + 0.02099447138607502, + -0.006167400162667036, + 0.013499334454536438, + 0.009721609763801098, + -0.024986913427710533, + 0.0033216208685189486, + 0.009534729644656181, + -0.015727916732430458, + 0.004858734086155891, + -0.004807835910469294, + 0.023319978266954422, + 0.0070745861157774925, + -0.0007084516109898686, + -0.0005870909662917256, + 0.06189145892858505, + 0.013250695541501045, + 0.022151706740260124, + 0.018919439986348152, + -0.030221926048398018, + 0.03543233498930931, + -0.014199546538293362, + -0.003100055968388915, + -0.011528171598911285, + 0.02346857078373432, + -0.005413527600467205, + 0.009707451798021793, + -0.003786312183365226, + 0.003278827527537942, + 0.008711902424693108, + -0.02070496417582035, + 0.005179683677852154, + 0.020465442910790443, + 0.011749315075576305, + -0.047200627624988556, + 0.017559422180056572, + -0.021039851009845734, + 0.0038782558403909206, + -0.0003238223434891552, + -0.006690016016364098, + 0.025282394140958786, + 0.01115462090820074, + 0.010405370034277439, + 0.027546733617782593, + -0.0047239987179636955, + -0.01912212185561657, + -0.019248036667704582, + 0.01046939566731453, + -0.010051258839666843, + -0.05386316776275635, + -0.014526155777275562, + 0.006520450580865145, + -0.05065187066793442, + 0.002640153979882598, + 0.01572658121585846, + -0.019353196024894714, + 0.00024770558229647577, + -0.002884111599996686, + -0.024859873577952385, + -0.03675568476319313, + 0.02946317195892334, + 0.009483127854764462, + 0.005995678249746561, + 0.018819313496351242, + -0.0010470274137333035, + -0.017008915543556213, + -0.009109362959861755, + 0.0038403330836445093, + -0.0357552170753479, + 0.003378189168870449, + 0.03931169584393501, + 0.008869672194123268, + -0.0033439104445278645, + 0.011275260709226131, + -0.018937986344099045, + 0.0015186145901679993, + -0.03934121131896973, + 0.008794485591351986, + -0.003144911490380764, + -0.004554149694740772, + -0.04801612347364426, + 0.033867157995700836, + -0.027917932718992233, + 0.0451841875910759, + -0.008013982325792313, + 0.00020571947970893234, + -0.008518164046108723, + 0.02732880972325802, + 0.0042418609373271465, + 0.028662150725722313, + -0.022526118904352188, + 0.006677933502942324, + 0.0233029592782259, + 0.014771399088203907, + -0.025703024119138718, + -0.015704622492194176, + -0.007107396610081196, + 0.03564446046948433, + -0.01832835003733635, + -0.03120797872543335, + 0.01848500967025757, + -0.0313805490732193, + 0.01430321391671896, + 0.0446428619325161, + -0.03135635703802109, + 0.04321533441543579, + 0.02104724571108818, + 0.02022634632885456, + 0.02186550572514534, + -0.004650760907679796, + -0.0033300602808594704, + 0.009849905036389828, + -0.02392319031059742, + -0.010538720525801182, + -0.022826816886663437, + -0.0065869493409991264, + -0.003717186162248254, + -0.003868849715217948, + -0.03139768913388252, + 0.03494035452604294, + -0.03037438541650772, + 0.03132350370287895, + -0.01891685463488102, + -0.0024421904236078262, + 0.02150551974773407, + -0.0010640721302479506, + -0.029242971912026405, + 0.0017518545500934124, + 0.01597743295133114, + -0.005137442145496607, + -0.009568083100020885, + 0.014652722515165806, + -0.04135658219456673, + -0.0007143799448385835, + 0.008337749168276787, + 0.042648620903491974, + 0.03566097840666771, + 0.012212038040161133, + 0.03211333602666855, + -0.0017624145839363337, + 0.030038531869649887, + 0.04514618590474129, + -0.04267878085374832, + 0.01551415491849184, + -0.007354247383773327, + 0.005497958045452833, + -0.002795785665512085, + 0.018108969554305077, + 0.02101774327456951, + -0.013560681603848934, + -0.011845670640468597, + -0.027034765109419823, + -0.028694113716483116, + -0.018704896792769432, + 0.043888069689273834, + -0.008286369033157825, + -0.029951654374599457, + 0.01802624762058258, + 0.013246898539364338, + 0.04284185916185379, + 0.0019446630030870438, + 0.01874862238764763, + -0.0034634629264473915, + 0.00524201150983572, + -0.02003074437379837, + 0.004723597317934036, + -0.002921441802754998, + 0.016102954745292664, + -0.008207092992961407, + -0.020983409136533737, + 0.024637114256620407, + -0.004268536809831858, + -0.021974073722958565, + 0.04152333363890648, + -0.024424513801932335 + ], + "attn_peak_rel": 0.0, + "attn_entropy": 1.7614775436880503e-10, + "attn_spread_pm2": 1.0, + "attn_role": "self", + "n_pairs": 8 + }, + "AGENT_PATIENT": { + "l_star": 16, + "h_dom": 15, + "snr": 5.695479385448738, + "specificity": 2.6144902563654338, + "canonical_vec": [ + -0.008591165766119957, + -0.03732172027230263, + -0.010591444559395313, + 0.04160040616989136, + -0.016931263729929924, + -0.0035179308615624905, + -0.010299918241798878, + -0.027807623147964478, + 0.041540395468473434, + -0.019560139626264572, + -0.007496888283640146, + 0.034312453120946884, + -0.042203355580568314, + 0.07017392665147781, + 0.028501365333795547, + -0.006051613483577967, + 0.036443814635276794, + 0.013994310051202774, + 0.02483205311000347, + -0.02705867402255535, + -0.026201102882623672, + 0.023003000766038895, + 0.03623560070991516, + 0.030026845633983612, + 0.01217703614383936, + -0.019208386540412903, + -8.620940934633836e-05, + 0.00509636988863349, + -0.05279483273625374, + -0.041866183280944824, + 0.007264427375048399, + -0.03036399371922016, + -0.03247721865773201, + -0.02014285698533058, + 0.0640467032790184, + -0.028948813676834106, + 0.008288634940981865, + 0.04013031721115112, + -0.010399582795798779, + 0.01868196204304695, + 0.03838522732257843, + -0.011492566205561161, + -0.020895062014460564, + -0.05012739449739456, + -0.016913145780563354, + -0.015017534606158733, + 0.029827870428562164, + 0.03077191673219204, + -0.03983468934893608, + 0.01550388801842928, + 0.025439836084842682, + 0.04434315115213394, + 0.01811111904680729, + 0.0011096441885456443, + 0.032363586127758026, + 0.060541871935129166, + 0.025429248809814453, + -0.005846803542226553, + 0.03464924171566963, + -0.023850515484809875, + 0.003149966709315777, + 0.04426425322890282, + -0.0667002946138382, + -0.049749862402677536, + 0.03922731429338455, + -0.008355235680937767, + -0.01916985586285591, + 0.010703982785344124, + -0.020347200334072113, + -0.020462656393647194, + 0.02477419748902321, + -0.0317537784576416, + -0.030312344431877136, + -0.01962685026228428, + 0.02182786539196968, + -0.005911353975534439, + 0.02928757853806019, + -0.026703637093305588, + 0.03705637902021408, + -0.019209733232855797, + -0.01626516692340374, + -0.004357524681836367, + -0.03553730249404907, + 0.05370517820119858, + -0.010701246559619904, + -0.0021932164672762156, + 0.006236083339899778, + -0.0025325713213533163, + -0.004157437011599541, + -0.021325135603547096, + 0.03509654849767685, + 0.012990223243832588, + -0.06954392045736313, + -0.024861175566911697, + -0.06930036097764969, + -0.015135596506297588, + 0.01139407604932785, + -0.0030336943455040455, + -0.012097638100385666, + 0.0348939523100853, + 0.06642840802669525, + 0.01637905277311802, + -0.04479819908738136, + 0.004255796317011118, + -0.019574427977204323, + -0.03518315404653549, + 0.0078119197860360146, + 0.014600400812923908, + -0.02683095447719097, + -0.00870970543473959, + 0.009239370934665203, + 0.024388844147324562, + 0.0032762980554252863, + -0.0674811452627182, + -0.03769760951399803, + 0.04177068918943405, + 0.01068043988198042, + 0.0034018107689917088, + 0.042408186942338943, + -0.05169565975666046, + -0.013722050003707409, + 0.013041626662015915, + 0.01914081536233425, + 0.007053832057863474, + -0.01110853161662817, + 0.008826229721307755, + 0.022128818556666374, + -0.0378773957490921, + 0.03510899469256401, + -0.014750299975275993, + 0.01837889663875103, + -0.0434432215988636, + -0.017258193343877792, + -0.0021146873477846384, + 0.002507111756131053, + -0.03811643272638321, + -0.040372174233198166, + 0.05046830698847771, + 0.006553342565894127, + 0.04408641904592514, + -0.010114777833223343, + 0.036225467920303345, + -0.0010334468679502606, + -0.03837123140692711, + 0.04621559754014015, + -0.021186675876379013, + 0.04504692554473877, + -6.466386253123346e-07, + 0.010218466632068157, + 0.022543184459209442, + 0.04776892811059952, + -0.016232596710324287, + -0.009044588543474674, + 0.03200583532452583, + -0.01622384786605835, + 0.021955134347081184, + -0.01567832939326763, + -0.006205512210726738, + 0.03024446591734886, + -0.02928094193339348, + -0.012268445454537868, + -0.016545213758945465, + -0.02439616434276104, + 0.0655374750494957, + 0.05003175139427185, + 0.03191915154457092, + -0.008337875828146935, + -0.01870741695165634, + -0.01785595342516899, + 0.0016099060885608196, + -0.01812497340142727, + -0.013816408812999725, + -0.005100875161588192, + 0.018305964767932892, + 0.019640950486063957, + 0.04555152356624603, + -0.0351308211684227, + 0.004983276594430208, + 0.014235815033316612, + -0.0019928759429603815, + 0.009726198390126228, + 0.018945254385471344, + 0.03507503122091293, + -0.030450204387307167, + 0.02198578417301178, + 0.008649714291095734, + -0.03373681753873825, + -0.02906871773302555, + 0.037880972027778625, + 0.017017202451825142, + 0.03544561564922333, + 0.0379294753074646, + 0.021156713366508484, + 0.04203784465789795, + 0.030692020431160927, + -0.018494145944714546, + -0.0030616039875894785, + 0.024718787521123886, + 0.007902727462351322, + 0.018050383776426315, + -0.01687326282262802, + -0.04651620611548424, + -0.019283516332507133, + 0.0367203988134861, + 0.05330998823046684, + 0.03321562334895134, + 0.007526436820626259, + -0.007149326615035534, + 0.04562833532691002, + 0.028607135638594627, + -0.03257225453853607, + -0.005512295290827751, + -0.05338073521852493, + -0.03914019092917442, + -0.058281589299440384, + -0.016434745863080025, + 0.028379730880260468, + -0.007139327935874462, + 0.04848482087254524, + 0.09397953748703003, + 0.02135254256427288, + -0.06117067113518715, + 0.05288759618997574, + 0.0076558091677725315, + -0.010538429953157902, + 0.0531497485935688, + 0.03244961053133011, + 0.05172941833734512, + 0.02637675032019615, + -0.034446801990270615, + 0.02704295516014099, + 0.038868848234415054, + -0.00937887467443943, + 0.047886449843645096, + -0.005271455738693476, + -0.003512526396661997, + 0.017396191135048866, + 0.021528271958231926, + 0.01845400221645832, + -0.015490819700062275, + -0.02555392123758793, + 0.016250107437372208, + -0.030012007802724838, + 0.014074799604713917, + 0.011778713203966618, + -0.038694608956575394, + 0.0006289758603088558, + 0.041974831372499466, + -0.01403763797134161, + -0.05658349767327309, + 0.0028533954173326492, + -0.02173568867146969, + -0.013507968746125698, + 0.0438261441886425, + 0.021665705367922783, + 0.0005588462227024138, + -0.003712414763867855, + -0.01172732561826706, + 0.02467872016131878, + -0.006167159881442785, + -0.016785718500614166, + -0.015214801765978336, + 0.044413913041353226, + -0.008031856268644333, + -0.003749020164832473, + -0.011996562592685223, + -0.020426513627171516, + -0.016515452414751053, + -0.02494092658162117, + 0.009945695288479328, + -0.024591881781816483, + -0.0032834268640726805, + 0.016427215188741684, + -0.0023586060851812363, + -0.03520498052239418, + 0.011980799958109856, + -0.005920737516134977, + 0.008911816403269768, + -0.012363207526504993, + -0.008205732330679893, + -0.04742615297436714, + -0.0023282296024262905, + 0.011754009872674942, + -0.016682730987668037, + -0.05304395407438278, + 0.0234846081584692, + 0.02619858644902706, + -0.01679392345249653, + 0.015059971250593662, + -0.02616063691675663, + -0.05481818690896034, + -0.04277437552809715, + 0.014667820185422897, + -0.023749232292175293, + 0.006435914896428585, + 0.02205057069659233, + 0.0174629595130682, + -0.03689773380756378, + 0.05104207992553711, + 0.00035693711834028363, + 0.04019969329237938, + 0.020340021699666977, + 0.003928573802113533, + -0.014109001494944096, + 0.035817116498947144, + 0.011035321280360222, + -0.020800644531846046, + -0.015508679673075676, + 0.04974006116390228, + -0.024771085008978844, + -0.04536692053079605, + 0.03193947300314903, + -0.07752562314271927, + -0.0070380461402237415, + -0.011650948785245419, + 0.03655959293246269, + 0.006477626506239176, + -0.024820253252983093, + 0.06592395901679993, + -0.01526745967566967, + -0.06947070360183716, + -0.002114057308062911, + -0.034308336675167084, + -0.022008854895830154, + -0.02022617682814598, + 0.03420387580990791, + 0.002161248354241252, + 0.00021519942674785852, + -0.02931893616914749, + 0.032230194658041, + -0.01815629005432129, + 0.0010791628155857325, + -0.04214676842093468, + -0.003008126514032483, + -0.06673100590705872, + 0.004555295221507549, + 0.012495062313973904, + 0.0169087965041399, + 0.009564027190208435, + -0.05037344619631767, + -0.023802582174539566, + -0.025409836322069168, + 0.023124095052480698, + 0.020300090312957764, + 0.01258345227688551, + 0.019626257941126823, + 0.0038530773017555475, + 0.021066240966320038, + -0.013726589269936085, + 0.026922238990664482, + 0.022463832050561905, + 0.026388056576251984, + 0.009715288877487183, + 0.01461833156645298, + -0.004330746829509735, + 0.04201068729162216, + -0.04581829160451889, + 0.013910500332713127, + 0.03033842146396637, + -0.04942992702126503, + 0.013469070196151733, + 0.02111186645925045, + 0.027527909725904465, + -0.029687397181987762, + 0.03895597532391548, + -0.029576338827610016, + 0.03725014626979828, + -0.017766354605555534, + -0.027720948681235313, + 0.0038652264047414064, + -0.020921654999256134, + 0.048784706741571426, + 0.0041658515110611916, + 0.09656504541635513, + -0.002875516889616847, + 0.04522836580872536, + -0.008045054040849209, + -0.00757100572809577, + -0.021853521466255188, + -0.025837615132331848, + 0.02398625947535038, + -0.01858198083937168, + -0.09248150140047073, + -0.04547916725277901, + -0.006209259387105703, + 0.020355209708213806, + -0.02099519409239292, + 0.009519224055111408, + 0.027229299768805504, + -0.003269640263170004, + 0.05124810338020325, + 0.025126192718744278, + -0.01709246076643467, + 0.015302193351089954, + 0.04773011803627014, + -0.030243879184126854, + 0.032915934920310974, + -0.07749543339014053, + -0.004318727646023035, + -0.05463520810008049, + -0.003882313147187233, + -0.05604472756385803, + 0.006672080140560865, + -0.007595330476760864, + 0.015355614945292473, + 0.02700590342283249, + 0.02027355320751667, + -0.0008797121117822826, + -0.012102468870580196, + 0.01694670133292675, + -0.016114363446831703, + 0.03402283415198326, + -0.04915110394358635, + -0.010459867306053638, + 0.09153337776660919, + 0.005748337134718895, + 0.0374646931886673, + -0.008943845517933369, + -0.018104929476976395, + 0.025005465373396873, + 0.028544887900352478, + 0.001093846745789051, + -0.017456037923693657, + -0.011575843207538128, + 0.0003019310242962092, + -0.054676253348588943, + 0.00755833275616169, + 0.006851560436189175, + 0.014164166525006294, + 0.056462522596120834, + -0.013366376049816608, + -0.006592321675270796, + -0.03411523252725601, + -0.057302750647068024, + -0.014713786542415619, + 0.022325459867715836, + -0.009898818098008633, + 0.03576203063130379, + 0.021434947848320007, + -0.006974563002586365, + -0.0035543309058994055, + 0.0025972190778702497, + 0.01461837999522686, + -0.014721233397722244, + 0.025337275117635727, + -0.007961557246744633, + 0.045258019119501114, + -0.020720139145851135, + 0.007749192416667938, + -0.04694071412086487, + -0.0005284174112603068, + 0.06214721500873566, + 0.03518009930849075, + -0.016371697187423706, + -0.03980552405118942, + -0.0390302836894989, + 0.0018615107983350754, + -0.001316952402703464, + 0.054515790194272995, + -0.007143115159124136, + 0.044627077877521515, + -0.008127952925860882, + -0.022252285853028297, + -0.005013479385524988, + 0.03328783065080643, + 0.04012708738446236, + -0.039905328303575516, + 0.06367972493171692, + 0.04611268639564514, + -0.03832917660474777, + 0.011785207316279411, + -0.0118589848279953, + 0.0747452974319458, + 0.001143609406426549, + -0.04751334339380264, + -0.01750904880464077, + -0.048293452709913254, + -0.0642278715968132, + -0.008085565641522408, + -0.04116188362240791, + 0.00372707424685359, + -0.011270338669419289, + -0.04133777320384979, + 0.02018517069518566, + -0.044363293796777725, + 0.03304608166217804, + 0.0029842532239854336, + -0.007552329450845718, + 0.016267159953713417, + -0.03730826452374458, + -0.002050400245934725, + 0.05075467377901077, + 0.027927063405513763, + 0.005947442259639502, + 0.0038764888886362314, + -0.05660058557987213, + 0.001793590490706265, + -0.038640979677438736, + 0.023263275623321533, + -0.00635266350582242, + -0.007958711124956608, + -0.028894338756799698, + 0.035301584750413895, + -0.011824307031929493, + 0.04742748662829399, + -0.006151415407657623, + 0.028925837948918343, + 0.0460125170648098, + 0.050493814051151276, + -0.0026057809591293335, + 0.005221632309257984, + -0.06329134851694107, + -0.007407031953334808, + 0.01753126084804535, + 0.030681757256388664, + -0.014386778697371483, + 0.019382411614060402, + 0.04126643389463425, + 0.009721074253320694, + 0.03290043771266937, + -0.026686623692512512, + 0.04284234717488289, + -0.0324578583240509, + -0.013851636089384556, + 0.02813054993748665, + -0.013653051108121872, + -0.0034192921593785286, + -0.017824625596404076, + -0.013350269757211208, + -0.018812667578458786, + 0.019859809428453445, + -0.02907748520374298, + -0.007273316849023104, + -0.053171563893556595, + 0.010147379711270332, + -0.06366118043661118, + 0.040414806455373764, + -0.008422869257628918, + 0.02975641004741192, + -0.023234156891703606, + 0.028338396921753883, + -0.030884042382240295, + -0.01538224145770073, + 0.0011261585168540478, + -0.05379728972911835, + 0.02318928763270378, + -0.08367744833230972, + -0.017835505306720734, + -0.008004446513950825, + -0.10338552296161652, + -0.008434724994003773, + -0.02171275019645691, + -0.0020296245347708464, + 0.0005024880520068109, + -0.009093452244997025, + -0.001414157566614449, + -0.02713584341108799, + 0.015286188572645187, + -0.06394211947917938, + -0.05117567628622055, + 0.006026380229741335, + -0.018721558153629303, + 0.06035532057285309, + -0.0024758512154221535, + 0.001847035251557827, + -0.0026132026687264442, + -0.022541608661413193, + 0.07352934032678604, + -0.012945597060024738, + -0.021102387458086014, + 0.04630133882164955, + -0.008840684778988361, + -0.017541801556944847, + -0.01295778900384903, + 0.017254073172807693, + 0.03030901961028576, + 0.039551880210638046, + -0.045780811458826065, + 0.00789901427924633, + -0.025242304429411888, + -0.028969256207346916, + 0.09106729924678802, + -0.03819047659635544, + 0.025601007044315338, + 0.008446614257991314, + 0.007509259507060051, + -0.036987170577049255, + -0.003940176218748093, + -0.016220543533563614, + -0.024994993582367897, + 0.06813017278909683, + -0.0049536325968801975, + 0.019826821982860565, + -0.031823545694351196, + 0.007920486852526665, + -0.028971970081329346, + 0.045308325439691544, + 0.0056437719613313675, + 0.01694050058722496, + 0.0025819677393883467, + 0.025100788101553917, + -0.0035961263347417116, + 0.002802331931889057, + 0.026086481288075447, + 0.02631382830440998, + -0.05006738752126694, + -0.000302018946968019, + -0.033532749861478806, + -0.003741646884009242, + -0.012320451438426971, + 0.039692047983407974, + -0.0070594726130366325, + -0.008212707936763763, + -0.014671637676656246, + -0.006646470166742802, + 0.03530571609735489, + -0.006485392339527607, + -0.025479057803750038, + 0.0011053651105612516, + -0.04423091933131218, + -0.036669228225946426, + 0.026599733158946037, + -0.023106234148144722, + 0.029024753719568253, + 0.04268153756856918, + 0.03064979799091816, + 0.014782854355871677, + 0.05386488884687424, + -0.012276748195290565, + 0.031237367540597916, + 0.01890772394835949, + -0.0004871547862421721, + -0.0040209004655480385, + -0.04100140184164047, + -0.03112676367163658, + -0.06498434394598007, + -0.022385289892554283, + -0.0007203879649750888, + -0.07014857232570648, + -0.02361300215125084, + 0.027705196291208267, + -0.029259251430630684, + 0.04479188844561577, + -0.016115857288241386, + -0.01999884471297264, + -0.026683343574404716, + 0.0015213263686746359, + -0.0084388367831707, + 0.037059105932712555, + -0.015115797519683838, + -0.017753664404153824, + 0.014725144021213055, + -0.06638975441455841, + -0.017498575150966644, + 0.036335427314043045, + 0.005253749433904886, + -0.016917197033762932, + -0.04063244163990021, + -0.014780418947339058, + -0.006700823083519936, + 0.00622931495308876, + 0.02750692330300808, + -0.02655797451734543, + -0.0014250176027417183, + -0.0006070489762350917, + 0.009431960061192513, + 0.018440447747707367, + 0.020880239084362984, + 0.013368864543735981, + -0.028789259493350983, + 0.0016244701109826565, + 0.003913586493581533, + 0.03896702080965042, + 0.046988289803266525, + 0.0013049576664343476, + -0.006717512849718332, + 0.04342230036854744, + 0.03677939996123314, + 0.010873901657760143, + -0.0014819707721471786, + 0.059807948768138885, + -0.021603982895612717, + 0.07218960672616959, + 0.026246177032589912, + -0.015302100218832493, + 0.01599641516804695, + -0.005234731361269951, + 0.011019213125109673, + 0.024169202893972397, + 0.02960243821144104, + 0.0036717825569212437, + -0.006484293844550848, + -0.024730755016207695, + 0.0743165984749794, + 0.042119983583688736, + 0.010635724291205406, + -0.021640028804540634, + 0.03149658814072609, + 0.00309858750551939, + 0.004087125416845083, + -0.03875475376844406, + 0.0010134093463420868, + -0.037201084196567535, + -0.03194218501448631, + -0.012106447480618954, + -0.02921799011528492, + -0.0026385767851024866, + 0.04207712784409523, + -0.004538408946245909, + -0.016945239156484604, + 0.07669050991535187, + -0.03922201320528984, + 0.033522579818964005, + 0.025370310992002487, + 0.020785856992006302, + 0.02364620752632618, + -0.06570025533437729, + 0.0033478315453976393, + -0.020198315382003784, + -0.056506894528865814, + -0.02400066703557968, + -0.03778867796063423, + 0.08594226092100143, + -0.027883516624569893, + -0.06594335287809372, + -0.00331689091399312, + -0.029302848502993584, + 0.006590292323380709, + -0.02197752706706524, + -0.08992008864879608, + -0.0070465137250721455, + -0.027191758155822754, + -0.02834301069378853, + -0.03563404455780983, + 0.0574033260345459, + -0.017772264778614044, + 0.016943059861660004, + 0.005732515826821327, + -0.003750424599274993, + 0.06014854088425636, + 0.041428424417972565, + 0.03702740743756294, + -0.006249873898923397, + -0.04003548249602318, + -0.04519682005047798, + 0.00668788468465209, + 0.032382622361183167, + -0.0037249636370688677, + 0.025248399004340172, + 0.05754053220152855, + 0.009766720235347748, + 0.03404868766665459, + -0.011968027800321579, + 0.021895894780755043, + -0.024535221979022026, + 0.01173054613173008, + -0.06407475471496582, + 0.009006407111883163, + -0.04151665419340134, + -0.013559030368924141, + -0.014846814796328545, + 0.06579410284757614, + -0.03548078611493111, + 0.02163468301296234, + 0.05577624589204788, + 0.003770810319110751, + 0.005505665205419064, + 0.03324286639690399, + -0.01435626670718193, + -0.019179420545697212, + -0.007731650024652481, + -0.056305695325136185, + -0.021509401500225067, + -0.06824308633804321, + -0.023314522579312325, + 0.001484894659370184, + 0.009756996296346188, + -0.03765913471579552, + -0.02680257521569729, + -0.014364742673933506, + 0.011489109136164188, + -0.013622472062706947, + 0.02535511925816536, + -0.005699935369193554, + -0.043636009097099304, + -0.02677510678768158, + -0.030212443321943283, + 0.030688337981700897, + -0.054927680641412735, + 0.02462410368025303, + -0.047190841287374496, + 0.06461482495069504, + -0.048203665763139725, + -0.03693414852023125, + -0.009175801649689674, + -0.014147244393825531, + -0.04334501922130585, + 0.036549147218465805, + -0.0012043376918882132, + 0.031861402094364166, + 0.00521271163597703, + 0.01958923414349556, + -0.0017771314596757293, + -0.033921968191862106, + 0.027776984497904778, + -0.02554110810160637, + 0.022016584873199463, + -0.053346969187259674, + 0.026845674961805344, + -0.02352270483970642, + -0.047544993460178375, + -0.03665894642472267, + 5.9948881244054064e-05, + -0.005039832089096308, + -0.021378984674811363, + -0.0175241120159626, + -0.05062442645430565, + 0.023493969812989235, + 0.049622420221567154, + 0.015026584267616272, + -0.0195828415453434, + -0.039706598967313766, + -0.015091425739228725, + 0.05921236425638199, + 0.02500041574239731, + 0.034209489822387695, + -0.005542072933167219, + -0.013609676621854305, + -0.006081261672079563, + 0.018228989094495773, + 0.03118467703461647, + 0.016090858727693558, + -0.019092727452516556, + -0.02538212202489376, + 0.015916144475340843, + -0.022943194955587387, + -0.04416351020336151, + 0.01093692984431982, + 0.02433799020946026, + -0.016355589032173157, + -0.02444739080965519, + 0.021106675267219543, + -0.041360318660736084, + 0.05696827918291092, + -0.007392410654574633, + -0.026452818885445595, + -0.01820974610745907, + -0.002278808504343033, + -0.005907618440687656, + -0.04471084848046303, + 0.0007563635590486228, + -0.0066153970547020435, + -0.030558254569768906, + -0.05508210137486458, + 0.06076691299676895, + 0.0276105348020792, + 0.044269662350416183, + -0.03379574045538902, + -0.017170889303088188, + -0.0461333729326725, + 0.015232706442475319, + -0.028515608981251717, + 0.005232463590800762, + -0.04492998123168945, + 0.05220099166035652, + -0.06042269989848137, + 0.007112478371709585, + -0.003962525632232428, + 0.04487703740596771, + 0.006418126635253429, + -0.048204392194747925, + -0.004463593475520611, + 0.016754774376749992, + 0.048476625233888626, + -0.09249606728553772, + -0.026146944612264633, + 0.0010763360187411308, + 0.026138540357351303, + -0.011793668381869793, + -0.05397079885005951, + 0.02443106658756733, + -0.024818994104862213, + -0.04840384051203728, + -0.014922390691936016, + 0.01017217431217432, + 0.011684461496770382, + 0.019891301169991493, + 0.06678163260221481, + 0.03230348974466324, + 0.03313026949763298, + 0.011201316490769386, + -0.02973361872136593, + 0.04141771420836449, + 0.024890111759305, + -0.04376181960105896, + -0.07059217244386673, + 0.006617802660912275, + -0.023346751928329468, + 0.04384620860219002, + 0.01245398074388504, + -0.017835885286331177, + -0.0037981721106916666, + -0.014807785861194134, + 0.015294237062335014, + 0.009595486335456371, + 0.02038687653839588, + 0.044237617403268814, + 0.02663431689143181, + 0.0035107426811009645, + 0.032930221408605576, + 0.032043710350990295, + -0.03993980586528778, + -0.007512644864618778, + -0.03518255427479744, + 0.053878773003816605, + 0.040161751210689545, + 0.039375241845846176, + 0.016934243962168694, + -0.003337799571454525, + 0.05588268116116524, + -0.007468703202903271, + -0.05670102313160896, + -0.02806810662150383, + 0.005453023128211498, + -0.018083637580275536, + 0.018027571961283684, + 0.030906863510608673, + 0.047201499342918396, + 0.051641449332237244, + 0.0010906518436968327, + -0.03358982875943184, + -0.035214293748140335, + 0.07676361501216888, + 0.035647034645080566, + -0.022076180204749107, + 0.03667063266038895, + -0.03119787760078907, + 0.017012469470500946, + 0.017062468454241753, + -0.0013652913039550185, + 0.03976447135210037, + 0.0174570195376873, + -0.019649004563689232, + 0.004420354031026363, + 0.0376974381506443, + 0.025612471625208855, + 0.04497801885008812, + 0.04542301967740059, + 0.006056095007807016, + -0.018909934908151627, + 0.029310686513781548, + -0.039017487317323685, + -0.001888384809717536, + -0.02195785753428936, + -0.006268012803047895, + -0.014043350704014301, + -0.0036608201917260885, + -0.014043251052498817, + -0.029371971264481544, + 0.03897577151656151, + 0.027118796482682228, + 0.01385182049125433, + 0.041554179042577744, + -0.01949680969119072, + -0.033750299364328384, + -0.02188294008374214, + 0.03146377205848694, + -0.01122600119560957, + 0.0016691080527380109, + -0.015999486669898033, + 0.08157507330179214, + -0.05435656011104584, + -0.017874404788017273, + 0.0016767262713983655, + 0.006920929998159409, + -0.017142141237854958, + -0.023208631202578545, + -0.005590359214693308, + 0.028637222945690155, + 0.0370633639395237, + 0.0267373975366354, + -0.01769225113093853, + -0.037065234035253525, + 0.0323852002620697, + -0.04619888588786125, + 0.04817059636116028, + 0.014941232278943062, + 0.017656998708844185, + 0.007173140067607164, + -0.003177542006596923, + 0.021914608776569366, + 0.01210109330713749, + 0.008989837020635605, + 0.013314281590282917, + -0.009666846133768559, + -0.024312997236847878, + 0.05050625652074814, + -0.003907360136508942, + 0.015261317603290081, + 0.037891656160354614, + 0.024683453142642975, + 0.032258789986371994, + -0.04918607696890831, + -0.01677633821964264, + 0.005788104142993689, + 0.008362679742276669, + -0.02390212006866932, + -0.006887413095682859, + -0.024408500641584396, + 0.011268841102719307, + -0.019799623638391495, + 0.030291149392724037, + -0.013529190793633461, + 0.0035615679807960987, + 0.005433754995465279, + -0.03399419039487839, + -0.0003124370414298028, + -0.005327693186700344, + -0.004683308303356171, + -0.02863621525466442, + -0.02718392387032509, + 0.0160142183303833, + -0.025925131514668465, + 0.014044920913875103 + ], + "attn_peak_rel": -1.0, + "attn_entropy": 0.060732610989362, + "attn_spread_pm2": 1.0, + "attn_role": "prev_token", + "n_pairs": 8 + }, + "QUANTITY_SCOPE": { + "l_star": 15, + "h_dom": 14, + "snr": 6.770835842994867, + "specificity": 4.070829735969339, + "canonical_vec": [ + 0.016393110156059265, + 0.028011804446578026, + -0.0001844739745138213, + 0.02673640474677086, + 0.025970203801989555, + -0.0027555502019822598, + -0.0006866104668006301, + -0.004425609018653631, + -0.0017879705410450697, + -0.20428717136383057, + 0.028595605865120888, + -0.021088125184178352, + 0.01469547301530838, + 0.015910577028989792, + 0.005827369634062052, + -0.00935136154294014, + -0.017150580883026123, + 0.004404985811561346, + -0.012356561608612537, + -0.012915006838738918, + -0.009697881527245045, + 0.027386842295527458, + 0.007532602176070213, + 0.017902957275509834, + -0.005211353302001953, + -0.02545783668756485, + -0.017639290541410446, + -0.035150978714227676, + 0.018221741542220116, + 0.011159965768456459, + 0.013119400478899479, + 0.011897662654519081, + 0.0008687081281095743, + 0.03147360682487488, + -0.020229602232575417, + 0.027462802827358246, + 0.013616818003356457, + 0.011000852100551128, + 0.006334206555038691, + 0.012506105937063694, + -0.014451487921178341, + 0.004406828433275223, + -0.005591634660959244, + 0.002276974031701684, + 0.026497732847929, + -0.017325880005955696, + 0.012047006748616695, + 0.021931977942585945, + 0.0010167342843487859, + -0.022285396233201027, + 0.010164852254092693, + 0.007465529255568981, + 0.004185238853096962, + 0.027684668079018593, + -0.04743943363428116, + 0.014668449759483337, + -0.0192645825445652, + -0.012899019755423069, + 0.024243541061878204, + 0.008223247714340687, + -0.021659627556800842, + -0.025029078125953674, + 0.030129000544548035, + -0.04326378554105759, + 0.0007646384183317423, + -0.014358887448906898, + -0.022394433617591858, + -0.027302704751491547, + 0.004057369194924831, + 0.020382605493068695, + 0.00037722039269283414, + 0.01382206566631794, + -0.000694971764460206, + 0.004815730731934309, + -0.052179522812366486, + 0.0012151330010965466, + -0.0007032056455500424, + 0.011387000791728497, + 0.012911658734083176, + -0.022232700139284134, + -0.01670726016163826, + -0.022089539095759392, + 0.004023347981274128, + 0.0028119098860770464, + -0.004947652108967304, + -0.006503192707896233, + -0.03372398763895035, + -0.015408056788146496, + 0.035202376544475555, + 0.004351083189249039, + 0.00472054211422801, + 0.007393935229629278, + -0.02399894781410694, + -0.00039325800025835633, + -0.02406064234673977, + 0.006035924423485994, + -0.00344146229326725, + -0.00585346482694149, + 0.02918844483792782, + 0.026979312300682068, + 0.00917714275419712, + 0.02947819232940674, + 0.01830918714404106, + -0.0027564570773392916, + -0.037424761801958084, + 0.04143281280994415, + 0.0008111604256555438, + 0.020680367946624756, + -0.002778285415843129, + -0.02052215300500393, + 0.01713995635509491, + 0.01392285991460085, + -0.0009021677542477846, + -0.02523794397711754, + 0.03159177675843239, + -0.016931552439928055, + 0.005189141258597374, + 0.008422210812568665, + 0.0035609197802841663, + 0.015429663471877575, + 0.006763298064470291, + 0.010921638458967209, + -0.031406648457050323, + 0.009236334823071957, + 0.0017297666054219007, + 0.009659417904913425, + 0.012925508432090282, + -0.018520668148994446, + -0.012083987705409527, + 0.005846836604177952, + 0.0014583267038688064, + -0.03381872549653053, + -0.011482750996947289, + 0.015807177871465683, + -0.01259923167526722, + 0.010379638522863388, + -0.027844151481986046, + -0.004269993398338556, + -0.019976256415247917, + 0.005788446869701147, + -0.0636562630534172, + 0.003262479091063142, + -0.0013813551049679518, + 0.007933766581118107, + -0.014270243234932423, + -0.014296814799308777, + -0.030486473813652992, + -0.01587107591331005, + -0.035425953567028046, + 0.007595979608595371, + -0.0018802621634677052, + 0.03409485146403313, + 0.010079773142933846, + 0.012648206204175949, + -0.028896069154143333, + 0.026135986670851707, + 0.0012732511386275291, + 0.04111234471201897, + 0.012699359096586704, + -0.005038878880441189, + -0.00419272668659687, + 0.041142046451568604, + -0.010371343232691288, + 0.0031435841228812933, + -0.02032299153506756, + -0.007619374431669712, + 0.006238413974642754, + -0.01595715992152691, + 0.004695377312600613, + 0.014092976227402687, + -0.011953328736126423, + 0.0008327535470016301, + -0.012890079990029335, + 0.012554454617202282, + -0.010366924107074738, + -0.002464546589180827, + -0.00898041296750307, + 0.00536706019192934, + 0.009259497746825218, + -0.02037644572556019, + 0.03380284830927849, + -0.01575472764670849, + -0.016206540167331696, + -0.028925733640789986, + -0.002469546627253294, + -0.0035389463882893324, + 0.00020365217642392963, + -0.01056906022131443, + -0.0004009748809039593, + 0.03319212794303894, + 0.02056838758289814, + 0.017463259398937225, + 0.021452059969305992, + 0.020905684679746628, + 0.025079263374209404, + 0.019483817741274834, + 0.004763412289321423, + 0.022781331092119217, + -0.002130320994183421, + -0.009335405193269253, + -0.015757333487272263, + -0.012844857759773731, + -0.010507182218134403, + 0.0010076678590849042, + -0.008266524411737919, + 0.005368215497583151, + 0.014159168116748333, + -0.021806353703141212, + 0.020481780171394348, + 0.011115591041743755, + 0.02154090814292431, + -0.001403407659381628, + 0.024137841537594795, + -0.010839985683560371, + -0.04402324557304382, + 0.004455882590264082, + -0.01641758158802986, + 0.017689146101474762, + -0.008390498347580433, + -0.003518776735290885, + 0.0037874814588576555, + -0.019584616646170616, + 0.016722921282052994, + -0.007134264800697565, + 0.0016134640900418162, + -0.008215726353228092, + 0.03464169427752495, + 0.009308847598731518, + 0.009932631626725197, + 0.01599150337278843, + -0.01311260275542736, + -0.035174984484910965, + 0.0013274474767968059, + 0.013627376407384872, + -0.004507336765527725, + -0.031988855451345444, + -0.02101941779255867, + 0.012243183329701424, + 0.004268260672688484, + 0.025562988594174385, + 0.00921901036053896, + -0.049167752265930176, + 0.011483232490718365, + -0.022985806688666344, + 0.004415007308125496, + -0.022801997140049934, + -0.005882607772946358, + 0.019081275910139084, + 0.004719337448477745, + 0.012089431285858154, + -0.027960505336523056, + 0.004376526921987534, + 0.0078064436092972755, + -0.02931777387857437, + -0.0005283149657770991, + -0.014788025990128517, + -0.0056403265334665775, + 0.0028326779138296843, + -0.02737339399755001, + -0.02921963669359684, + -0.012939558364450932, + 0.008649609982967377, + 0.00024358009977731854, + 0.002055106684565544, + 0.02244389057159424, + -0.04259374365210533, + 0.015071016736328602, + -0.016009576618671417, + -0.7637245655059814, + -0.024345170706510544, + 0.048953764140605927, + -0.013639342039823532, + 0.016235962510108948, + -0.010141991078853607, + 0.01244460791349411, + 0.01062733307480812, + 0.00402708537876606, + -0.010852934792637825, + 0.001759909326210618, + 0.007474243640899658, + -0.009933006949722767, + 0.024315517395734787, + -0.003243109444156289, + 0.019797399640083313, + 0.03483708202838898, + -0.008846704848110676, + 0.033357515931129456, + 0.002051508752629161, + 0.012344860471785069, + 0.011332734487950802, + -0.0281059630215168, + 0.025516247376799583, + 0.012457084842026234, + -0.03972511366009712, + 0.004570432007312775, + 0.00201153545640409, + 0.003921076655387878, + -0.004199085757136345, + 0.02732033096253872, + -0.007814276963472366, + 0.015626294538378716, + 0.008251420222222805, + -0.020378930494189262, + 0.0014773341827094555, + -0.004169589839875698, + 0.002708208514377475, + -0.016267545521259308, + 0.013244304805994034, + 0.02594251185655594, + 0.01663762889802456, + 0.040132537484169006, + -0.015867309644818306, + -0.030584337189793587, + -0.016041019931435585, + -0.044566575437784195, + 0.01614401862025261, + 0.002121267607435584, + 0.0408378429710865, + 0.007136758882552385, + -0.0010123872198164463, + 0.029716314747929573, + 0.011019255965948105, + 0.02115625888109207, + -0.005936420522630215, + 0.008008038625121117, + 0.009274711832404137, + -0.0023488651495426893, + -0.01345368567854166, + 0.015017567202448845, + -0.018917948007583618, + 0.0015516204293817282, + 0.0037089139223098755, + 0.005228533875197172, + 0.010929589159786701, + 0.0022145931143313646, + -0.026098882779479027, + -0.015971951186656952, + 0.017181703820824623, + -0.00483908224850893, + -0.028246846050024033, + 0.00961464922875166, + 0.029999280348420143, + 0.030705945566296577, + -0.023281864821910858, + 0.015866216272115707, + -0.051113326102495193, + 0.017340855672955513, + 0.008638652041554451, + 0.008007606491446495, + -0.011686410754919052, + 0.005286651197820902, + -0.011814706027507782, + 0.016190096735954285, + -0.006854390725493431, + -0.031483858823776245, + 0.012021520175039768, + 0.05125992372632027, + -0.010475155897438526, + 0.017417943105101585, + -0.008803119882941246, + -0.01001347042620182, + -0.013482526876032352, + 0.01135933492332697, + 0.014271206222474575, + 0.015637893229722977, + 0.004129740409553051, + -0.004636324010789394, + 0.010565291158854961, + 0.004596462938934565, + -0.0025741588324308395, + 0.013014192692935467, + 0.02083035185933113, + 0.024231091141700745, + -0.002017356688156724, + -0.010928128845989704, + -0.014259785413742065, + -0.0374206118285656, + -0.024734949693083763, + -0.010181056335568428, + 0.024904469028115273, + 0.01504648756235838, + -0.006098067853599787, + 0.005985272116959095, + 0.021137317642569542, + -0.006790525745600462, + -0.007652441039681435, + 0.014906193129718304, + 0.043954022228717804, + 0.006112085189670324, + 0.013845509849488735, + 0.007967113517224789, + -0.012388608418405056, + 0.0005143253947608173, + -0.038364388048648834, + 0.031209886074066162, + -0.013211689889431, + -0.024841655045747757, + -0.05945175513625145, + -0.013372013345360756, + -0.04077926278114319, + 0.01148315705358982, + 0.008248048834502697, + -0.032460395246744156, + -0.01032540574669838, + 0.021680431440472603, + 0.007086345925927162, + 0.031358931213617325, + -0.018247190862894058, + 0.010864438489079475, + 0.006145452614873648, + -0.01908310316503048, + -0.0016103357775136828, + 0.02122843638062477, + -0.02158387377858162, + -0.009894120506942272, + 0.028279457241296768, + 0.0010157912038266659, + -0.0031490109395235777, + -0.019086312502622604, + -0.010672906413674355, + -0.025137269869446754, + 0.0013298973208293319, + -0.004613080061972141, + 0.003109435783699155, + 0.01978648267686367, + 0.0018052997766062617, + 0.016081156209111214, + 0.016616174951195717, + 0.003463964443653822, + -0.02124810591340065, + 0.03219194337725639, + -0.005624026525765657, + -0.00140950211789459, + -0.024655906483530998, + 0.02013520896434784, + -0.0014045577263459563, + -0.028660671785473824, + 0.0023827608674764633, + 0.00038379657780751586, + -0.018828002735972404, + 0.0016341577284038067, + -0.006719871424138546, + 0.01764727383852005, + 0.020824376493692398, + -0.0281443540006876, + 0.02126336097717285, + -0.03647653013467789, + -0.012526677921414375, + 0.004121392499655485, + 0.02930576354265213, + -0.013799019157886505, + -0.009405368007719517, + -0.007038837298750877, + -0.011721753515303135, + -0.016899261623620987, + 0.010561184026300907, + 0.018632421270012856, + 0.0291132852435112, + 0.013186924159526825, + -0.03390849009156227, + -0.05036698281764984, + -0.04330657050013542, + 0.004513032268732786, + -0.008669290691614151, + 0.01579686626791954, + -0.004061031620949507, + -0.026418020948767662, + -0.011617887765169144, + -0.0031440467573702335, + 0.030081983655691147, + 0.012525074183940887, + -0.00041890950524248183, + -0.005835663061589003, + -0.034115102142095566, + 0.010056022554636002, + -0.004124762490391731, + 0.01786961406469345, + -0.040921613574028015, + -0.00267046969383955, + -0.0195146631449461, + -0.021190272644162178, + 0.026653053238987923, + 0.02001609094440937, + 0.03965366259217262, + -0.017260035499930382, + 0.02823159284889698, + -0.019975967705249786, + -0.0032481770031154156, + -0.010756179690361023, + -0.008831829763948917, + -0.012125340290367603, + -0.021222196519374847, + 7.255843229359016e-05, + 0.0037594286259263754, + 0.012716829776763916, + -0.020440734922885895, + 0.02596973441541195, + 0.0004945528926327825, + -0.002161092823371291, + 0.03342472016811371, + -0.008056080900132656, + 0.007960382848978043, + 0.010348272509872913, + -0.012344416230916977, + -0.037491414695978165, + 0.004401053301990032, + 0.01407396700233221, + -0.03510661423206329, + 0.007926801219582558, + -0.03402251377701759, + 0.009178846143186092, + -0.00507313571870327, + -0.01947753131389618, + 0.010872025974094868, + 0.02741856873035431, + 0.0047331624664366245, + 0.039324674755334854, + -0.019605709239840508, + 0.013087539002299309, + 0.013081615790724754, + 0.02599913254380226, + -0.00994638167321682, + 0.0036222354974597692, + 0.009163868613541126, + 0.005360569804906845, + 0.008645391091704369, + 0.008035375736653805, + 0.0029000546783208847, + 0.007350319530814886, + -0.0005318949115462601, + -0.03378702327609062, + 0.0039013768546283245, + -0.04639216884970665, + 0.0033741367515176535, + -0.00025017300504259765, + 0.015021199360489845, + -0.015319878235459328, + -0.01328249741345644, + -0.0061471350491046906, + -0.011281873099505901, + 0.010017049498856068, + 0.008775546215474606, + 0.027189748361706734, + 0.007114876061677933, + 0.005680573172867298, + -0.022262629121541977, + 0.01477847807109356, + -0.002262095222249627, + 0.018740201368927956, + -0.05025537684559822, + -0.021179087460041046, + -0.033024195581674576, + 0.010905851610004902, + -0.030809184536337852, + -0.017881957814097404, + 0.006232342682778835, + 0.004002775531262159, + 0.00456279655918479, + 0.023947009816765785, + -0.04766254127025604, + 0.006493020337074995, + 0.017905186861753464, + -0.01770358346402645, + -0.012470434419810772, + -0.009829249233007431, + 0.004650456365197897, + -0.010812096297740936, + -0.005138710606843233, + 0.005629778373986483, + 0.002401125617325306, + -0.016325443983078003, + 0.009146232157945633, + 0.013880643993616104, + -0.008671271614730358, + -0.011341231875121593, + 0.014757675118744373, + -0.012602136470377445, + -0.03331131860613823, + 0.019045744091272354, + -0.0029713353142142296, + 0.010699785314500332, + 0.010357506573200226, + -0.01751014217734337, + -0.062038131058216095, + 0.02311701886355877, + 0.001122501795180142, + 0.004186046309769154, + 0.0044362954795360565, + 0.01890767738223076, + -0.001961721107363701, + 0.018386855721473694, + 0.009389219805598259, + -0.0059440769255161285, + -0.00023062511172611266, + -0.0005785446846857667, + 0.018113331869244576, + -0.0034511475823819637, + -0.04276815429329872, + 0.0109846917912364, + -0.023835791274905205, + -0.023873982951045036, + 0.03580990061163902, + 0.005712352693080902, + 0.021564817056059837, + -0.017944974824786186, + -0.037307851016521454, + -0.01624378189444542, + 0.0024749336298555136, + 0.008051445707678795, + -0.001883552991785109, + 0.013573677279055119, + 0.0005708635435439646, + 0.0006540719186887145, + -0.022562680765986443, + 0.0057585956528782845, + 0.014207103289663792, + -0.009719419293105602, + 0.013447343371808529, + 0.01372437085956335, + 0.002030058531090617, + -0.007532672490924597, + -0.011981437914073467, + -0.0024590427055954933, + 0.014184203930199146, + -0.0022276851814240217, + 0.011469992808997631, + 0.0037126722745597363, + 0.010394372045993805, + -0.004045096691697836, + -0.00801105983555317, + 0.016322972252964973, + -0.010419628582894802, + -0.010398362763226032, + -0.0021995946299284697, + 0.028730522841215134, + -0.003241549013182521, + 0.006550896447151899, + 0.016849564388394356, + 0.011491534300148487, + 0.0075515154749155045, + -0.03859298303723335, + -0.029304753988981247, + 0.002353872638195753, + -0.012770583853125572, + -0.0044854688458144665, + 0.007918567396700382, + -0.004421787802129984, + 0.017346912994980812, + 0.01136639341711998, + -0.021862415596842766, + 0.003562516998499632, + 0.018552226945757866, + 0.005098008085042238, + 0.0049706147983670235, + 0.009288587607443333, + -0.009955095127224922, + 0.009331723675131798, + 0.0484408475458622, + 0.008796540088951588, + -0.02580486796796322, + 0.009535438381135464, + 0.02428123727440834, + -0.0023078834637999535, + -0.017744308337569237, + -0.004344030283391476, + 0.014356245286762714, + -0.0018658458720892668, + 0.015365135855972767, + 0.010254111140966415, + 0.002654407871887088, + 0.0020273346453905106, + -0.03416360914707184, + 0.004704765975475311, + 0.019271697849035263, + 0.0009230544092133641, + -0.012776759453117847, + -0.00538340350612998, + 0.01164599321782589, + 0.0003319471434224397, + 0.019667044281959534, + 0.019561568275094032, + 0.013447066769003868, + -0.03348877653479576, + 0.014637729153037071, + 0.014801690354943275, + -0.024912802502512932, + 0.01349056325852871, + 0.004999858792871237, + -0.02577100694179535, + -0.012179250828921795, + -0.02641574665904045, + 0.05943480134010315, + -0.020141130313277245, + -0.02053723856806755, + -0.002741762902587652, + 0.00427077803760767, + -0.01754859834909439, + -0.03247891739010811, + -0.009778736159205437, + -0.011828637681901455, + 0.0003221263468731195, + 0.00630100816488266, + 0.035018496215343475, + -0.056416526436805725, + 0.005916880909353495, + 0.008360330015420914, + -0.0010047894902527332, + 0.020140916109085083, + 0.0003692159370984882, + -0.0018218703335151076, + 0.05077460780739784, + -0.008699600584805012, + 0.019825242459774017, + -0.003566344967111945, + 0.02088019624352455, + -0.005619308911263943, + -0.0326971709728241, + 0.03055710904300213, + 0.0289249736815691, + 0.003187863389030099, + -0.004274964332580566, + -0.021891461685299873, + -0.010351082310080528, + 0.007316265255212784, + 0.0083905765786767, + 0.0027531373780220747, + -0.008634076453745365, + 0.007535742130130529, + 0.027759959921240807, + 0.03526917099952698, + -0.03178421035408974, + 0.02719135768711567, + -0.009441949427127838, + 0.02654036320745945, + 0.016916444525122643, + -0.005559517070651054, + 0.03356890007853508, + -0.004130830522626638, + 0.012740238569676876, + -0.02436676248908043, + -0.03658419847488403, + 0.04750356078147888, + -0.02314828522503376, + 0.03654451668262482, + -0.01992739923298359, + -0.02051006443798542, + -0.026147665455937386, + 0.004213028587400913, + -0.019110077992081642, + -0.0011480925604701042, + 0.03569610044360161, + 0.01651301607489586, + -0.020475059747695923, + 0.01663191430270672, + 0.015587163157761097, + -0.009575934149324894, + 0.034245897084474564, + 0.001514612347818911, + 0.0023535070940852165, + -0.005613576155155897, + -0.0013504145899787545, + 0.006969341542571783, + -0.021539680659770966, + -0.0026972959749400616, + 0.0038439687341451645, + -0.022129185497760773, + -0.025936126708984375, + 0.0078111509792506695, + -0.004372275900095701, + 0.023153018206357956, + 0.010955102741718292, + -0.012689769268035889, + -0.014024781994521618, + -0.016377843916416168, + 0.00605367124080658, + 0.026775911450386047, + 0.013566641137003899, + 0.003255259245634079, + -0.03380145877599716, + 0.005731879733502865, + 0.003897009650245309, + 0.005791258532553911, + 0.008522615768015385, + -0.037604331970214844, + -0.03170142322778702, + -0.021328207105398178, + -0.025180242955684662, + 0.013947512023150921, + 0.018105916678905487, + -0.02498408406972885, + -0.028405213728547096, + -0.017059287056326866, + 0.0058692279271781445, + -0.003660984104499221, + 0.028099754825234413, + 0.0001889093837235123, + -0.0031942524947226048, + 0.02142137661576271, + -0.0041038901545107365, + 0.009763206355273724, + 0.007692055776715279, + -0.010872820392251015, + 0.029349200427532196, + 0.03339293971657753, + -0.020191965624690056, + 0.03662155568599701, + 0.004960655700415373, + -0.022755805402994156, + 0.0022668514866381884, + 9.945406054612249e-05, + 0.002503148512914777, + -0.0022032689303159714, + 0.004799752030521631, + 0.02345276065170765, + -0.01137547567486763, + -0.016075046733021736, + -0.018741896376013756, + -0.02863817662000656, + 0.002381386235356331, + 0.02442139759659767, + 0.014084978960454464, + -0.031864043325185776, + 0.014849708415567875, + -0.0013635107316076756, + -0.0007574621122330427, + -0.013507273979485035, + 0.026866182684898376, + 0.0025302781723439693, + -0.004277520813047886, + -0.022183727473020554, + 0.007370735984295607, + 0.040418293327093124, + -0.008621522225439548, + 0.02641536481678486, + -0.01138053648173809, + -0.03216937929391861, + -0.013920523226261139, + -0.002500406699255109, + -0.023992469534277916, + -0.022870292887091637, + -0.006397131830453873, + -0.016219332814216614, + 0.0009912220994010568, + -0.025083178654313087, + 0.050745394080877304, + 0.024722646921873093, + 0.01670745015144348, + -0.002610723488032818, + -0.01941854879260063, + 0.004101551603525877, + 0.00593978725373745, + -0.02274247817695141, + -0.02969375252723694, + 0.03926311433315277, + -0.034965697675943375, + 0.005439640488475561, + 0.0004884542431682348, + -0.01493498682975769, + 0.036747924983501434, + 0.0052080606110394, + 0.01253743190318346, + 0.005823230370879173, + -0.015165760181844234, + -0.01043782290071249, + 0.02936348132789135, + -0.027418386191129684, + 0.010501976124942303, + -0.024217676371335983, + 0.016889432445168495, + 0.005134605336934328, + 0.04761013016104698, + 0.013479477725923061, + -0.029071355238556862, + -0.03262439742684364, + 0.013409487903118134, + -0.0048809717409312725, + 0.004843565635383129, + -0.02551177702844143, + 0.04525018483400345, + 0.0036665964871644974, + -0.005799711681902409, + -0.017181215807795525, + -0.012153349816799164, + -0.003682028502225876, + 0.037973761558532715, + -0.015884794294834137, + 0.012251553125679493, + 0.012174426577985287, + 0.0080028111115098, + 0.017798149958252907, + 0.00934517476707697, + -0.02521633170545101, + -0.025960955768823624, + -0.000881821964867413, + -0.018022798001766205, + -0.01133219338953495, + 0.009037778712809086, + 0.0017578911501914263, + -0.01966671086847782, + -0.001392526552081108, + -0.024726850911974907, + -0.02031080052256584, + -0.006043521221727133, + 0.0005741537897847593, + 0.0025571768637746572, + -0.01737343519926071, + 0.001802125945687294, + 0.01495585311204195, + -0.01877697929739952, + -0.014598001725971699, + 0.005373244173824787, + -0.02710573375225067, + 0.00673278933390975, + -0.03218939155340195, + -0.011298737488687038, + -0.006367525085806847, + 0.014527172781527042, + -0.005369172897189856, + -0.019290978088974953, + 0.008052321150898933, + -0.015091377310454845, + 0.0034187869168817997, + -0.0001655363303143531, + 0.040480535477399826, + 0.034929703921079636, + -0.04024245962500572, + 0.0068388511426746845, + -0.006931360345333815, + -0.002446400700137019, + -0.0020048795267939568, + -0.008609892800450325, + 0.006819617003202438, + 0.014866428449749947, + 0.006751419976353645, + -0.010255412198603153, + 0.028720417991280556, + 0.007956335321068764, + 0.006733819842338562, + -0.007584681734442711, + -0.004301036242395639, + -0.0011805564863607287, + 0.005574394017457962, + 0.007680482231080532, + -0.018907440826296806, + 0.028971854597330093, + -0.044888898730278015, + 0.014524215832352638, + 0.019079910591244698, + 0.01029990240931511, + 0.0044647883623838425, + 0.0036109096836298704, + -0.05655204504728317, + 0.028150729835033417, + -0.0057470062747597694, + -0.005854769144207239, + -0.0032596595119684935, + -0.008912173099815845, + 0.003380922367796302, + 0.019334372133016586, + 0.005359059199690819, + -0.0026346491649746895, + -0.0068626124411821365, + 0.008806879632174969, + -0.0051293326541781425, + 0.030308669432997704, + 0.023692306131124496, + 0.027777893468737602, + 0.008747966028749943, + 0.0318961925804615, + -0.006649889517575502, + -0.004470078274607658, + 0.027411315590143204, + 0.009589827619493008, + -0.0011263759806752205, + 0.03383259475231171, + 0.001936033135280013, + -0.005313802044838667, + 0.0026084710843861103, + -0.010387055575847626, + 0.00869653932750225, + -0.03509557619690895, + -0.013394242152571678, + 0.002461634576320648, + 0.003827093867585063, + 0.045093122869729996, + -0.013705352321267128, + 0.0042801289819180965, + -0.01096142828464508, + 0.006016518920660019, + 0.02637576498091221, + -0.0167622659355402, + 0.009860901162028313, + -4.270151475793682e-05, + -0.010183483362197876, + 0.032439958304166794, + 0.00573377450928092, + 0.00019915607117582113, + -0.0031159960199147463, + 0.027986770495772362, + -0.027470801025629044, + 0.006515695713460445, + -0.006367128808051348, + -0.014153974130749702, + 0.028258366510272026, + -0.025815801694989204, + -0.013522530905902386, + -0.004110487177968025, + -0.000562182511202991, + 0.013904464431107044, + 0.008191335014998913, + 0.0366780087351799, + -0.012777874246239662, + 0.008266359567642212, + -0.002157671609893441, + 0.006230310071259737, + -0.010462841019034386, + 0.003154323436319828, + -0.04119673743844032, + 0.03785305470228195, + -0.01667989231646061, + 0.000996231916360557, + -0.00601534079760313, + 0.001201133825816214, + 0.0005394621985033154 + ], + "attn_peak_rel": 0.0, + "attn_entropy": 1.1743183595674944e-10, + "attn_spread_pm2": 1.0, + "attn_role": "self", + "n_pairs": 8 + }, + "EPISTEMIC_CERTAINTY": { + "l_star": 8, + "h_dom": 15, + "snr": 1.4973230756547182, + "specificity": 1.4087603419057713, + "canonical_vec": [ + 0.025225626304745674, + 0.013959593139588833, + 0.01877247542142868, + -0.016556061804294586, + -0.023322002962231636, + -0.016570376232266426, + 0.05770960822701454, + -0.015919208526611328, + 0.042514413595199585, + 0.008287688717246056, + 0.02187841758131981, + 0.03920046240091324, + 0.0029952116310596466, + 0.021251752972602844, + -0.04408981278538704, + 0.04891984537243843, + 0.006652756128460169, + 0.045214731246232986, + 0.006907960399985313, + 0.013589122332632542, + -0.021623866632580757, + -0.001201192382723093, + -0.0007090377039276063, + -0.02915637008845806, + 0.02230064570903778, + 0.017257604748010635, + 0.03665231168270111, + -0.016086651012301445, + 0.010140856727957726, + 0.04764401167631149, + -0.008381057530641556, + 0.02134396694600582, + -0.005457893013954163, + -0.03158653900027275, + -0.056790441274642944, + -0.016560547053813934, + 0.05583130195736885, + -0.0036475323140621185, + -0.07074562460184097, + 0.01621651090681553, + -0.028495367616415024, + -0.03053942508995533, + -0.04080789163708687, + -0.02364359050989151, + 0.012811210937798023, + 0.04128338024020195, + -0.055229395627975464, + -0.07172911614179611, + 0.007953749969601631, + 0.028355984017252922, + 0.01124770287424326, + -0.015834135934710503, + -0.011037551797926426, + 0.039064109325408936, + -0.024766579270362854, + 0.04570358991622925, + -0.009599778801202774, + 0.015457573346793652, + -0.048640672117471695, + -0.008863101713359356, + 0.011707733385264874, + 0.034221772104501724, + 0.03849868103861809, + -0.02433975785970688, + -0.03101542964577675, + -0.031505562365055084, + -0.01952490583062172, + -0.003446338465437293, + -0.036786191165447235, + -0.09074684977531433, + -0.009287252090871334, + 0.03784587234258652, + -0.020016446709632874, + -0.016647540032863617, + 0.033350974321365356, + 0.0008900491520762444, + 0.07093407213687897, + -0.026642736047506332, + -0.026822784915566444, + 0.0006535383290611207, + -0.011666118167340755, + 0.03460961580276489, + 0.025454804301261902, + -0.013764760456979275, + 0.024194931611418724, + 0.04405365139245987, + 0.003938802983611822, + -0.03299020603299141, + 0.0510173998773098, + 0.0010531845036894083, + 0.018310213461518288, + 0.007772248238325119, + 0.010525419376790524, + -0.01010233722627163, + -0.05984516069293022, + 0.04060528054833412, + -0.0014073355123400688, + 0.012118804268538952, + 0.025477973744273186, + 0.015821829438209534, + -0.02568146400153637, + 0.04039296135306358, + 0.023124830797314644, + -0.026205938309431076, + 0.008854161016643047, + 0.0008387939305976033, + 0.013232041150331497, + 0.05966099724173546, + -0.0009721465758047998, + -0.011964108794927597, + -0.01777515560388565, + -0.03291374072432518, + -0.0018050499493256211, + 0.0012267600977793336, + 0.011570359580218792, + 0.009625403210520744, + -0.00264071486890316, + -0.03327653557062149, + 0.025904538109898567, + -0.01846853457391262, + -0.04690328985452652, + 0.0030794087797403336, + 0.017393028363585472, + -0.010355501435697079, + -0.013945442624390125, + 0.012002530507743359, + -0.0076981838792562485, + -3.3783300750656053e-05, + 0.004984584171324968, + 0.0007453833823092282, + -0.01939607597887516, + -0.01307287160307169, + -0.009401773102581501, + 0.014094579964876175, + -0.049327362328767776, + -0.008133133873343468, + 0.038774143904447556, + -0.03377476707100868, + -0.022859565913677216, + 0.025897860527038574, + -0.0210875291377306, + -0.015128403902053833, + -0.03614316135644913, + -0.018558837473392487, + 0.022172965109348297, + 0.007360909599810839, + 0.024865783751010895, + 0.02574910596013069, + -0.030603421851992607, + 0.00933687761425972, + -2.2144764443510212e-05, + -0.0065949102863669395, + -0.024461885914206505, + -0.009087637066841125, + -0.023229775950312614, + 0.07181588560342789, + -0.004146154969930649, + 0.002128235064446926, + 0.006606206297874451, + -0.051363758742809296, + 0.029735011979937553, + -0.0005208386573940516, + -0.012825754471123219, + -0.031716808676719666, + 0.019985634833574295, + -0.03274784982204437, + -0.02788573130965233, + -0.0044792527332901955, + 0.023655809462070465, + -0.009629814885556698, + 0.029800428077578545, + 0.038199737668037415, + 0.005904436577111483, + 0.02292240597307682, + -0.025226376950740814, + -0.0388752743601799, + 0.051129695028066635, + -0.041715823113918304, + -0.00256333127617836, + -0.0009883134625852108, + 0.028009489178657532, + -0.03657778352499008, + 0.02714681439101696, + -0.052825216203927994, + 0.012077745981514454, + -0.05223752185702324, + 0.02441859431564808, + -0.02583264373242855, + -0.03269876167178154, + -0.05084043741226196, + -0.03421242907643318, + -0.003250851295888424, + -0.014779401011765003, + 0.008027942851185799, + -0.003863309044390917, + 0.017794232815504074, + -0.052303560078144073, + -0.007558436132967472, + 0.03056582435965538, + 0.0395909883081913, + -0.0255789402872324, + -0.0013903952203691006, + 0.0377512164413929, + 0.0023518905509263277, + -0.01843847520649433, + 0.028666501864790916, + -0.028987083584070206, + -0.003816750133410096, + -0.001864633639343083, + 0.003934219479560852, + 0.013315746560692787, + -0.0034088639076799154, + -0.004997462034225464, + -0.0013064324157312512, + -0.021512873470783234, + 0.011518720537424088, + 0.0022817726712673903, + 0.008855365216732025, + -0.031041117385029793, + 0.013062152080237865, + -0.03680611029267311, + -0.06484808772802353, + -0.00999926496297121, + -0.029558120295405388, + -0.029553938657045364, + 0.035758014768362045, + -0.024698078632354736, + -0.004337241407483816, + -0.025095971301198006, + 0.002809038618579507, + -0.012264619581401348, + -0.008259906433522701, + 0.035433944314718246, + 0.03528304398059845, + -0.006076272577047348, + 0.05990558862686157, + 0.014176592230796814, + -0.008048869669437408, + 0.0005743862129747868, + -0.04195915162563324, + 0.012640129774808884, + 0.006528513506054878, + 0.0199733953922987, + -0.02037937007844448, + -0.013143371790647507, + 0.035706669092178345, + -0.06119533255696297, + -0.004127060994505882, + -0.027289915829896927, + 0.03372647985816002, + 0.04032155126333237, + 0.03006955049932003, + -0.04769285395741463, + 0.0025284155271947384, + 0.0538012757897377, + 0.006286601070314646, + -0.009869367815554142, + 0.011236405931413174, + -0.013961007818579674, + -0.0038746646605432034, + -0.03681265935301781, + -0.06889481097459793, + 0.0008043832494877279, + -0.0500609315931797, + 0.020830899477005005, + 0.02066088281571865, + -0.03442775830626488, + 0.03404190391302109, + 0.08216593414545059, + 0.030811525881290436, + -0.04393276572227478, + 0.029421504586935043, + 0.018420353531837463, + 0.02437940612435341, + -0.014127428643405437, + 0.033823829144239426, + -0.010834945365786552, + 0.01662387326359749, + -0.0035840938799083233, + 0.013074922375380993, + -0.014016346074640751, + -0.021744297817349434, + 0.0284595750272274, + -0.0483822338283062, + 0.021152028813958168, + 0.0554281547665596, + 0.0028479797765612602, + 0.008802101947367191, + -0.007131526712328196, + 0.022242149338126183, + 0.032852351665496826, + -0.011742061004042625, + -0.04511032626032829, + 0.04297550395131111, + 0.01367266196757555, + 0.016823409125208855, + 0.010821489617228508, + 0.002800597343593836, + -0.021120518445968628, + 0.025154778733849525, + -0.003927560523152351, + 0.016479965299367905, + -0.04232137277722359, + -0.03294149041175842, + 0.007464532740414143, + 0.021545255556702614, + 0.048777952790260315, + -0.012812210246920586, + 0.04356655851006508, + -0.03306444734334946, + 0.04368684068322182, + 0.043486740440130234, + -0.009864749386906624, + 0.0379193089902401, + 0.08681703358888626, + -0.02836381085216999, + -0.0002587230992503464, + -0.006524756550788879, + -0.007354372180998325, + -0.023420264944434166, + -0.01766734942793846, + 0.049602754414081573, + 0.023667261004447937, + 0.02417929656803608, + 0.0033466301392763853, + -0.03218361735343933, + 0.0025044968351721764, + 0.02571517415344715, + 0.02435939572751522, + 0.02147008292376995, + -0.0012031479272991419, + 0.02133903093636036, + -0.03567047417163849, + -0.02175992727279663, + 0.03445792570710182, + -0.06853663176298141, + 0.0820220336318016, + -0.025873921811580658, + 0.05589480325579643, + -0.029586974531412125, + 0.022210367023944855, + 0.04293841868638992, + -0.028422275558114052, + -0.03464464843273163, + 0.023770641535520554, + -0.019207073375582695, + -0.028761593624949455, + -0.01337374560534954, + 0.021730268374085426, + 0.02485528029501438, + 0.021417943760752678, + -0.029172837734222412, + 0.002266790485009551, + 0.011303815990686417, + -0.030483875423669815, + -0.02159843221306801, + 0.045443013310432434, + -0.0272196214646101, + 0.01749938353896141, + 0.0078807407990098, + -0.013128219172358513, + -0.015618526376783848, + 0.0046686134301126, + -0.004002925008535385, + 0.01006275787949562, + -0.025589080527424812, + 0.028706522658467293, + -0.04462910816073418, + -0.029689274728298187, + 0.03696814179420471, + -0.008710549212992191, + 0.008428907953202724, + 0.03765825554728508, + 0.014882399700582027, + 0.040981534868478775, + -0.03919491171836853, + 0.05087694525718689, + 0.008037093095481396, + 0.03242778405547142, + -0.04168442636728287, + 0.00310123385861516, + -0.05225251242518425, + -0.054550573229789734, + 0.03564612939953804, + 0.028513092547655106, + 0.017959939315915108, + -0.06085887923836708, + -0.007369600236415863, + 0.016816217452287674, + -0.0030863366555422544, + -0.03220188617706299, + -0.026513053104281425, + -0.00044643395813181996, + -0.01997983455657959, + -0.05066671594977379, + -0.022955702617764473, + -0.006495912559330463, + 0.004771022591739893, + -0.015524627640843391, + -0.00628858245909214, + -0.020878778770565987, + -0.04989103600382805, + -0.01826491951942444, + 0.015223829075694084, + 0.07231482118368149, + 0.033674657344818115, + -0.045421477407217026, + -0.016558866947889328, + -0.03336776793003082, + 0.018282609060406685, + 0.05517158657312393, + -0.004091538488864899, + 0.019688408821821213, + 0.07321817427873611, + -0.04351961612701416, + -0.07349107414484024, + 0.023661933839321136, + -0.015800198540091515, + 0.04491417482495308, + -0.007229409646242857, + 0.016502171754837036, + 0.008508548140525818, + 0.02647647075355053, + 0.0301930271089077, + -0.020420249551534653, + -0.02585121989250183, + -0.02908218465745449, + 0.008480495773255825, + -0.002688114996999502, + -0.013809849508106709, + 0.007890409789979458, + -0.009599457494914532, + 0.050030339509248734, + -0.06337695568799973, + -0.01980544440448284, + -0.007243988569825888, + -0.046634040772914886, + -0.06409118324518204, + 0.008262097835540771, + 0.026968467980623245, + -0.016198089346289635, + 0.011773473583161831, + -0.016887811943888664, + 0.015806037932634354, + 0.00837304163724184, + 0.019607840105891228, + -0.0319514200091362, + -0.0962185189127922, + 0.0035689149517565966, + -0.04717610031366348, + 0.06359174102544785, + -0.0366261824965477, + 0.04408612102270126, + 0.0017771139973774552, + 0.006314157508313656, + -0.03902943804860115, + -0.012771433219313622, + 0.02308555878698826, + -0.045379914343357086, + 0.01425880752503872, + -0.022687086835503578, + -0.010699131526052952, + 0.014590919949114323, + -0.06930572539567947, + 0.0023936107754707336, + -0.0031438739970326424, + -0.014934317208826542, + -0.023401686921715736, + -0.020363563671708107, + 0.023099960759282112, + 0.03644714504480362, + -0.007141020614653826, + -0.007206745445728302, + 0.027614975348114967, + -0.002316504716873169, + 0.01297722477465868, + -0.016612384468317032, + -0.011189277283847332, + -0.06337060779333115, + -0.036670565605163574, + -0.0012816701782867312, + -0.004629494156688452, + -0.04159269854426384, + 0.023790348321199417, + -0.0229048989713192, + 0.001504956977441907, + 0.0024857856333255768, + -0.07215207070112228, + 0.059989456087350845, + -0.05066925659775734, + -0.011921712197363377, + 0.057847052812576294, + 0.0003453900571912527, + -0.02829018421471119, + 0.06173185631632805, + -0.00746370991691947, + -0.011378156021237373, + 0.002762574004009366, + -0.01638689450919628, + 0.03516057878732681, + 0.005190194118767977, + -0.013112912885844707, + 0.010012633167207241, + 0.05602698400616646, + 0.027838045731186867, + 0.08493535965681076, + -0.06767654418945312, + 0.030453089624643326, + -0.06625072658061981, + -0.015960462391376495, + -0.022322019562125206, + -0.061592359095811844, + -0.017231278121471405, + 0.01963762938976288, + -0.004291112534701824, + -0.03206288814544678, + -0.03112189844250679, + 0.02842618152499199, + 0.016580944880843163, + 0.001942849950864911, + 0.027062321081757545, + -0.040909286588430405, + 0.02465399168431759, + -0.01962132379412651, + 0.04942566901445389, + 0.07816556096076965, + 0.0279157143086195, + 0.013261063024401665, + -0.022081414237618446, + -0.004062553867697716, + -0.010650932788848877, + 0.00023512440384365618, + 0.013783871196210384, + 0.010117103345692158, + 0.005547808017581701, + -0.013744635507464409, + -0.016695182770490646, + -0.02796551026403904, + 0.0027534998953342438, + -0.036442652344703674, + -0.006175998132675886, + 0.007240994833409786, + 0.020755145698785782, + 0.02990374155342579, + 0.03404329717159271, + -0.016995785757899284, + 0.02934546023607254, + 0.022802073508501053, + -0.0391894094645977, + -0.02333030477166176, + -0.04633764177560806, + 0.00700465589761734, + 0.05450477451086044, + -0.028774065896868706, + -0.030331159010529518, + -0.009225660935044289, + -0.01187137607485056, + 0.0061802938580513, + -0.01033048052340746, + -0.030973032116889954, + 0.04080454632639885, + -0.02094269171357155, + 0.017473744228482246, + 0.022381560876965523, + 8.05346790002659e-05, + -0.023741155862808228, + -0.00022478864411823452, + -0.03362910449504852, + -0.039831504225730896, + -0.02385776862502098, + 0.032071709632873535, + -0.049632806330919266, + 0.05187792703509331, + -0.004336541518568993, + -0.012966948561370373, + 0.04306936636567116, + -0.0334097295999527, + -0.06537842750549316, + 0.04141579940915108, + 0.005483897868543863, + 0.02083430625498295, + -0.012136432342231274, + -0.029298953711986542, + 0.05114765465259552, + 0.014544075354933739, + -0.0701911672949791, + -0.0233842134475708, + 0.001750955474562943, + 0.022987643256783485, + 0.050689004361629486, + -0.026070373132824898, + 0.009267951361835003, + -0.015511127188801765, + -0.045009464025497437, + -0.020018093287944794, + 0.056245796382427216, + -0.004801648203283548, + 0.01484257634729147, + -0.031839121133089066, + -0.012881290167570114, + 0.008023090660572052, + 0.01047187577933073, + 0.006391409318894148, + -0.018747780472040176, + -0.001543651451356709, + -0.018763011321425438, + -0.0045862579718232155, + -0.012417311780154705, + 0.027045952156186104, + 0.05182851478457451, + 0.05306851491332054, + 0.021653197705745697, + 0.006620117928832769, + -0.030118457973003387, + 0.011769816279411316, + -0.024502526968717575, + 0.029095228761434555, + -0.012104304507374763, + 0.0029345951043069363, + 0.012570397928357124, + -0.028909441083669662, + -0.04463794082403183, + 0.0263078436255455, + 0.020354973152279854, + -0.033570922911167145, + -0.009820246137678623, + -0.010730710811913013, + 0.005970044527202845, + 0.012210559099912643, + -0.0001885691162897274, + -0.02357928454875946, + 0.010281655006110668, + -0.005703524220734835, + 0.03381434828042984, + 0.025108929723501205, + 0.040857285261154175, + 0.012120922096073627, + -0.02503439411520958, + 0.0018522896571084857, + -0.018114490434527397, + -0.040635380893945694, + 0.03229495510458946, + -0.023448219522833824, + -0.017536073923110962, + -0.07718754559755325, + 0.010049494914710522, + -0.01156497374176979, + -0.08517581224441528, + 0.0012801584089174867, + -0.03738511726260185, + 0.005551799200475216, + -0.061245568096637726, + -0.046530164778232574, + 0.011273183859884739, + -0.01167906541377306, + -0.00016559411596972495, + -0.009615861810743809, + -0.01853875443339348, + -0.004823158960789442, + 0.06914860755205154, + -0.0026582819409668446, + 0.05243723839521408, + 0.004321001004427671, + -0.0001314049441134557, + 0.05822942778468132, + 0.025769947096705437, + 0.02031642571091652, + 0.010562197305262089, + -0.007294951472431421, + 0.016004299744963646, + -0.03439490124583244, + 0.043762777000665665, + 0.07304453104734421, + 0.0035175441298633814, + -0.0036152969114482403, + 0.03644992783665657, + 0.008961644023656845, + 0.02574814110994339, + 0.013379289768636227, + -0.006539086811244488, + 0.036775436252355576, + 0.04714072123169899, + 0.05222520977258682, + -0.02410968206822872, + -0.039219677448272705, + 0.020370779559016228, + -0.0018056818516924977, + -0.024370450526475906, + 0.00014997433754615486, + -0.02876855805516243, + 0.003188474802300334, + -0.023760728538036346, + -0.058592289686203, + 0.03610261529684067, + 0.06541983038187027, + -0.006907864939421415, + 0.058978091925382614, + 0.010047397576272488, + 0.023824190720915794, + -0.045795489102602005, + 0.02714552730321884, + 0.023244455456733704, + 0.012979851104319096, + 0.03493227809667587, + -0.010199186392128468, + -0.02138829417526722, + -0.008217550814151764, + 0.04318492114543915, + 0.023790037259459496, + -0.023612143471837044, + 0.028375590220093727, + 0.010088000446557999, + -0.035130199044942856, + -0.0180828794836998, + 0.027662279084324837, + 0.002466624602675438, + -0.007768254727125168, + 0.03113050013780594, + 0.03952959552407265, + 0.0001153633784269914, + -0.0012991692638024688, + 0.037137966603040695, + 0.025677183642983437, + 0.005709629971534014, + 0.02886902540922165, + 0.01664767600595951, + -0.018431957811117172, + -0.010897179134190083, + -0.004104842431843281, + 0.03310725837945938, + 0.055414117872714996, + -0.014565818943083286, + -0.010689901188015938, + -0.027982914820313454, + 0.01200411282479763, + 0.018916813656687737, + 0.0108752166852355, + -0.002202365780249238, + 0.019181054085493088, + 0.004496060311794281, + -0.020946886390447617, + -0.009041241370141506, + 0.029105011373758316, + 0.07516971975564957, + 0.011154236271977425, + 0.013253914192318916, + 0.013418849557638168, + 0.019641809165477753, + -0.02338561601936817, + 0.0018063423922285438, + 0.024956263601779938, + -0.011778274551033974, + 0.0553138442337513, + 0.006589855998754501, + -0.015707630664110184, + -0.03553340211510658, + -0.007438520900905132, + 0.024132635444402695, + -0.030817139893770218, + 0.052657946944236755, + 0.01924949698150158, + 0.03989892080426216, + -0.03689845651388168, + -0.022974373772740364, + -0.0160604789853096, + 0.00823826901614666, + -0.008358940482139587, + 0.02139444090425968, + 0.031012432649731636, + -0.020773207768797874, + -0.0035034464672207832, + -0.034220512956380844, + 0.06363768130540848, + 0.030260704457759857, + 0.024398308247327805, + 0.00812012143433094, + -0.03246130421757698, + -0.005890505854040384, + 0.027602870017290115, + 0.07425507158041, + 0.004228649660944939, + -0.01228525023907423, + -0.03187207505106926, + 0.029245464131236076, + 0.020177971571683884, + -0.015554673969745636, + 0.02719835937023163, + 0.023444661870598793, + -0.010788606479763985, + -0.024656085297465324, + 0.03109540417790413, + -0.018474387004971504, + -0.02560236304998398, + -0.046400561928749084, + 0.007630267180502415, + 0.04037046805024147, + 0.001155469217337668, + 0.01473755482584238, + 0.0103420065715909, + -0.08770407736301422, + 0.002884700195863843, + 0.020465359091758728, + -0.013268131762742996, + -0.01492796465754509, + 0.042237475514411926, + 0.0006996029987931252, + 0.002398357493802905, + 0.02310100384056568, + -0.037246670573949814, + -0.008337425999343395, + -0.0009423638111911714, + -0.03599851205945015, + 0.010603109374642372, + 0.028259750455617905, + -0.05512849614024162, + 0.021101314574480057, + -0.005823882762342691, + 0.012534275650978088, + -0.03640015795826912, + -0.012942517176270485, + -0.00997734721750021, + -0.016364818438887596, + 0.0023262244649231434, + -0.0008516390225850046, + -0.013919963501393795, + -0.027134457603096962, + 0.020785242319107056, + -0.03160702437162399, + -0.028313301503658295, + -0.0842939093708992, + 0.053085166960954666, + 0.047441501170396805, + 0.0073023405857384205, + 0.042649541050195694, + 0.012101328931748867, + 0.006568650249391794, + 0.017385665327310562, + -0.016612429171800613, + -0.010844042524695396, + 0.030229128897190094, + -0.031837042421102524, + 0.03359530493617058, + 0.028731411322951317, + -0.03889180347323418, + 0.06026618182659149, + -0.02431124821305275, + 0.08461125195026398, + -0.017770908772945404, + -0.005092250648885965, + 0.0598883330821991, + 0.02923215553164482, + -0.005354626104235649, + -0.012303312309086323, + 0.015367608517408371, + 0.03432392701506615, + 0.026051148772239685, + 0.015383320860564709, + 0.008078613318502903, + -0.03406969830393791, + 0.04261552914977074, + 0.02082795277237892, + -0.018082931637763977, + -0.021447181701660156, + -0.028707407414913177, + 0.031397104263305664, + -0.030480720102787018, + 0.03681325167417526, + 0.028272654861211777, + 0.05007733404636383, + 0.017241116613149643, + 0.028382768854498863, + -0.015331312082707882, + -0.023493139073252678, + -0.03331900015473366, + 0.0020220086444169283, + -0.02556665986776352, + -0.02721458114683628, + 0.011821587570011616, + 0.025615330785512924, + -0.0031465506181120872, + 0.002621303079649806, + 0.00402801251038909, + -0.03088255226612091, + 0.021930478513240814, + -0.03818623349070549, + 0.009920904412865639, + -0.007756536360830069, + -0.025312552228569984, + 0.02032146044075489, + -0.023744983598589897, + 0.0028880981262773275, + -0.05648688226938248, + 0.011294765397906303, + -0.040928926318883896, + 0.05677219480276108, + 0.0860663503408432, + -0.0016997044440358877, + -0.03598539158701897, + 0.008337154053151608, + -0.060670800507068634, + -0.031062547117471695, + 0.017815442755818367, + 0.04311218857765198, + -0.006046671885997057, + 0.03458303585648537, + -0.024310702458024025, + 0.011571151204407215, + -0.08644036948680878, + 0.022299768403172493, + 0.014154566451907158, + 0.025954538956284523, + -0.020244432613253593, + -0.003250689245760441, + -0.04729235917329788, + -0.04601667448878288, + -0.05370232090353966, + -0.024494308978319168, + -0.017358453944325447, + -0.001129471929743886, + 0.019841032102704048, + 0.047274742275476456, + 0.0301795806735754, + 0.028681030496954918, + 0.04301216080784798, + -0.009477236308157444, + -0.020414259284734726, + 0.05507305637001991, + 0.013675770722329617, + -0.04997929558157921, + -0.0006886845803819597, + 0.017128273844718933, + -0.021915266290307045, + -0.014449088834226131, + 0.011931853368878365, + -0.006053967867046595, + -0.024301793426275253, + 0.09539393335580826, + -0.013986186124384403, + 0.0021624709479510784, + -0.04722552374005318, + 0.007784613873809576, + -0.09531369060277939, + 0.031147096306085587, + 0.057846128940582275, + -0.017237458378076553, + 9.275185561818944e-07, + -0.013395684771239758, + 0.045310795307159424, + 0.0012692793970927596, + -0.06109810620546341, + 0.07993308454751968, + -0.022110063582658768, + -0.04133599251508713, + 0.05469886586070061, + 0.026429403573274612, + 0.03487097844481468, + 0.0086045628413558, + -0.03549922630190849, + 0.004483786411583424, + 0.008412792347371578, + -0.040086083114147186, + 0.044866591691970825, + 0.03235883638262749, + 0.004056497476994991, + 0.049246132373809814, + 0.025400683283805847, + 0.04006959870457649, + -0.004953454714268446, + -0.09776531904935837, + -0.007180442567914724, + 0.030124370008707047, + -0.056931380182504654, + 0.0010825031204149127, + -0.03528251871466637, + -0.017121143639087677, + 0.032006341964006424, + -0.030361758545041084, + -0.04105955734848976, + 0.07136081159114838, + -0.01677858456969261, + -0.00695073464885354, + 0.06399044394493103, + 0.022931858897209167, + 0.005161801353096962, + -0.04932607710361481, + 0.009628085419535637, + 0.0027438977267593145, + -0.03642229363322258, + 0.011112372390925884, + 0.018319593742489815, + 0.023715419694781303, + -0.0068585434928536415, + -0.09399337321519852, + -0.00957596953958273, + 0.05890130251646042, + 0.009637311100959778, + -0.039094291627407074, + -0.06667757034301758, + -0.0039011943154037, + -0.014973186887800694, + 0.01618092693388462, + -0.009783677756786346, + 0.017145365476608276, + -0.02714770846068859, + 0.017826346680521965, + -0.01926177181303501, + 0.022834982722997665, + -0.0385374017059803, + 0.03195182606577873, + 0.005525294225662947, + 0.007422256749123335, + 0.0010758636053651571, + 0.04869954660534859, + 0.03566577285528183, + -0.012396642938256264, + -0.04075906425714493, + -0.032887887209653854, + -0.027491271495819092, + 0.033842019736766815, + 0.013621671125292778, + -0.0050911796279251575, + -0.03894278034567833, + 0.010797790251672268, + -0.009118197485804558 + ], + "attn_peak_rel": -1.625, + "attn_entropy": 0.30437026359140873, + "attn_spread_pm2": 0.9999999850988388, + "attn_role": "looks_back", + "n_pairs": 8 + }, + "PART_WHOLE": { + "l_star": 19, + "h_dom": 13, + "snr": 2.032056497436996, + "specificity": 1.5801902609813272, + "canonical_vec": [ + -0.010661798529326916, + 0.002273125108331442, + -0.03445348143577576, + 0.019807808101177216, + -0.03327014297246933, + 0.007714741863310337, + 0.004429388791322708, + -0.025020018219947815, + -0.059997692704200745, + 0.00010010090045398101, + -0.03515470400452614, + 0.003447196213528514, + 0.02450224757194519, + -0.04521515592932701, + -0.03367329016327858, + -0.0097592081874609, + 0.006029085721820593, + 0.019544489681720734, + 0.035406507551670074, + 0.034939806908369064, + -0.011403452605009079, + 0.024935197085142136, + 0.023455552756786346, + 0.02133028209209442, + 0.01466077659279108, + 0.05490180104970932, + 0.013554186560213566, + 0.0029140159022063017, + -0.035789575427770615, + 0.0065573956817388535, + -0.057473454624414444, + -0.003702926216647029, + -0.005756949540227652, + -0.03285146877169609, + -0.006660482380539179, + -0.02287156693637371, + -8.124317537294701e-05, + 0.07415422052145004, + 0.014958905056118965, + -0.041561856865882874, + -0.0020836852490901947, + 0.0516357384622097, + 0.07280201464891434, + 0.05032431706786156, + 0.019835909828543663, + 0.05781799182295799, + 0.007248138077557087, + -0.008831032551825047, + -0.01401889231055975, + -0.02655969187617302, + -0.009385643526911736, + 0.005852502305060625, + -0.00815256405621767, + 0.05243661627173424, + -0.04183152690529823, + -0.03718161582946777, + -0.02881249226629734, + 0.013262408785521984, + -0.03696818649768829, + -0.053062804043293, + -0.012674489058554173, + 0.045085608959198, + 0.009938252158463001, + 0.04787508770823479, + -0.019867416471242905, + 3.7619738577632234e-05, + 0.03655219078063965, + 0.01811501570045948, + 0.008054450154304504, + 0.02064172364771366, + 0.026709530502557755, + 0.03298165276646614, + 0.01511538214981556, + 0.001823571277782321, + 0.015163588337600231, + -0.02439842000603676, + -0.01797998696565628, + -0.015224030241370201, + 0.005496187601238489, + -0.03251456841826439, + 0.03861190006136894, + -0.009538285434246063, + 0.023991914466023445, + 0.07338864356279373, + 0.000798222201410681, + -0.005165879148989916, + 0.07501085847616196, + 0.03939873352646828, + -0.031104790046811104, + -0.016572900116443634, + -0.02093532681465149, + 0.025122372433543205, + -0.0658724382519722, + -0.013961362652480602, + 0.019201863557100296, + -0.040742598474025726, + 0.023282768204808235, + 0.00996337179094553, + 0.011053995229303837, + -0.014179295860230923, + 0.02344484254717827, + 0.042637284845113754, + 0.03286248445510864, + 0.009523400105535984, + -0.028867453336715698, + -0.045807380229234695, + 0.0321553498506546, + -0.026657694950699806, + 0.04834131523966789, + -0.022592756897211075, + 0.01464565098285675, + -0.052386574447155, + 0.012629305012524128, + 0.050200022757053375, + -0.020349908620119095, + -0.006328303832560778, + 0.00019985446124337614, + 0.03325192630290985, + 0.04254252463579178, + -0.025182286277413368, + 0.04530108720064163, + -0.02904517576098442, + 0.01080709882080555, + 0.052715446799993515, + 0.05530424416065216, + -0.008561012335121632, + -0.0349310003221035, + 0.03856324777007103, + -0.05050426721572876, + 0.05227554216980934, + -0.0007945313700474799, + 0.014349967241287231, + -0.0013917373726144433, + 0.009195255115628242, + -0.020325230434536934, + 0.021707046777009964, + -6.735546048730612e-05, + -0.020035719498991966, + 0.038891080766916275, + -0.0381181575357914, + -0.00493389368057251, + -0.07095997780561447, + 0.018736766651272774, + 0.007798272185027599, + -0.05696415156126022, + -0.002006454858928919, + 0.05747837573289871, + -0.055146168917417526, + -0.041837528347969055, + -0.0007627845043316483, + -0.003373830346390605, + -0.02312609739601612, + -0.016771964728832245, + 0.0018331026658415794, + 0.00036218835157342255, + -0.015237461775541306, + 0.040909670293331146, + 0.007201160304248333, + 0.02987552620470524, + 0.023543035611510277, + 0.022197047248482704, + 0.03583034127950668, + 0.004399266093969345, + -0.04097612202167511, + -0.007351120002567768, + 0.0005748139228671789, + 0.0580892488360405, + -0.05711740255355835, + 0.012819252908229828, + 0.007565880659967661, + 0.018092690035700798, + -0.029581308364868164, + 0.019765961915254593, + -0.05595530569553375, + -0.006495865527540445, + -0.010752009227871895, + 0.05244206264615059, + -0.0380161851644516, + 0.039430275559425354, + -0.04021596908569336, + -0.03548288345336914, + 0.0004492767620831728, + 0.018058115616440773, + 0.04244711250066757, + -0.02161332592368126, + -0.03777872398495674, + 0.010424572974443436, + -0.034196171909570694, + -0.0663672462105751, + -0.026917526498436928, + -0.03077518753707409, + 0.031487490981817245, + -0.00016158785729203373, + -0.01923358626663685, + -0.0006403761217370629, + 0.03735225647687912, + 0.028383566066622734, + 0.03154519572854042, + 0.030178768560290337, + 0.0038448399864137173, + 0.021406082436442375, + -0.02133076824247837, + -0.030548645183444023, + 0.06622213125228882, + 0.020323345437645912, + 0.014325969852507114, + -0.024522215127944946, + -0.021465573459863663, + 0.04144319146871567, + 0.0031863495241850615, + -0.006204389967024326, + -0.01144582498818636, + 0.008917472325265408, + -0.02363763004541397, + 0.032999586313962936, + -0.06109144911170006, + 0.013655014336109161, + -0.022960778325796127, + -0.014949120581150055, + -0.02762770466506481, + 0.01521468535065651, + -0.005317590665072203, + 0.0004839825560338795, + 0.047118861228227615, + -0.05297619104385376, + -0.06048477441072464, + -0.031099289655685425, + -0.003411069978028536, + -0.01665741577744484, + -0.043555233627557755, + 0.05690407380461693, + -0.042569395154714584, + -0.007960817776620388, + -0.015647366642951965, + -0.03519135341048241, + -0.039331626147031784, + 0.06075796112418175, + -2.7375248464522883e-05, + -0.001204121159389615, + 0.07152188569307327, + 0.007390883751213551, + -0.08869349211454391, + 0.01948193646967411, + 0.01303996704518795, + -0.022365611046552658, + 0.003593161702156067, + -0.04395812377333641, + -0.03425433859229088, + -0.06585431098937988, + -0.00988785270601511, + -0.05133960396051407, + 0.011552758514881134, + -0.033494506031274796, + 0.009566589258611202, + 0.020442107692360878, + 0.026211703196167946, + -0.05417930707335472, + -0.005440270062536001, + -0.008185421116650105, + 0.05024591088294983, + -0.013467147946357727, + 0.034834250807762146, + 0.014424600638449192, + -0.004845879506319761, + 0.03442836180329323, + -0.08042577654123306, + -0.07942506670951843, + -0.013756089843809605, + 0.18453745543956757, + -0.008243893273174763, + -0.008261114358901978, + -0.018685869872570038, + -0.053113359957933426, + -0.040935590863227844, + -0.02801472134888172, + 0.030387843027710915, + -0.030804911628365517, + -0.02195759117603302, + -0.04643988609313965, + 0.0032487690914422274, + -0.013778993859887123, + 0.010742143727838993, + 0.0014014499029144645, + -0.04956844821572304, + 0.0066414475440979, + -0.017975104972720146, + 0.03008609265089035, + 0.032838962972164154, + -0.031941208988428116, + -0.04671662673354149, + -0.02954009920358658, + 0.008565470576286316, + 0.002472911262884736, + 0.05524539202451706, + 0.0033634284045547247, + 0.002235937165096402, + 0.003170676762238145, + -0.013185715302824974, + 0.01828937977552414, + -0.007038699928671122, + -0.051515087485313416, + 0.02444474585354328, + -0.014205386862158775, + 0.023931333795189857, + -0.014400498010218143, + 0.005756072234362364, + -0.016992557793855667, + -0.005879181902855635, + 0.008412923663854599, + 0.015122632496058941, + -0.018360896036028862, + 0.00774787925183773, + 0.03279883414506912, + -0.033430494368076324, + 0.002197555499151349, + 0.03319928050041199, + -0.001885530655272305, + -0.029197512194514275, + -0.0076469844207167625, + 0.006498298142105341, + 0.0428563728928566, + -0.04400906339287758, + -0.013287493959069252, + 0.060391176491975784, + -0.05732221528887749, + -0.023175427690148354, + -0.02993403933942318, + 0.014056752435863018, + 0.00038962429971434176, + -0.023632541298866272, + 0.015982873737812042, + 0.02284170687198639, + -0.004112910479307175, + -0.019881632179021835, + -0.07151059806346893, + 0.0568772628903389, + 0.012817970477044582, + 0.007073854561895132, + -0.029666049405932426, + 0.005435469094663858, + 0.004221667069941759, + -0.010427950881421566, + 0.026472583413124084, + -0.031856127083301544, + 0.05137002468109131, + -0.031074747443199158, + 0.04054874926805496, + -0.001554822432808578, + -0.031289249658584595, + 0.017697205767035484, + -0.07405244559049606, + 0.0059175267815589905, + 0.017606504261493683, + -0.019209418445825577, + -0.025348810479044914, + 0.030404195189476013, + -0.0015581396874040365, + 0.0468900240957737, + 0.0568041130900383, + 0.01740439422428608, + 0.022472161799669266, + 0.033360786736011505, + -0.024715309962630272, + -0.0033976705744862556, + -0.007495156489312649, + 0.010136093944311142, + -0.00975871179252863, + -0.0289625134319067, + 0.02858825773000717, + 0.048257745802402496, + 0.005887123290449381, + -0.043790753930807114, + -0.03426516428589821, + 0.02931780181825161, + -0.01226949505507946, + 0.023418154567480087, + 0.02842179872095585, + 0.010417112149298191, + -0.013653844594955444, + 0.021911976858973503, + 0.020635923370718956, + -0.001449262723326683, + 0.01427051704376936, + 0.023328520357608795, + 0.01340296771377325, + 0.043064359575510025, + 0.03010225109755993, + 0.00787199568003416, + -0.009719429537653923, + 0.016387946903705597, + 0.002971798414364457, + 0.0013813661644235253, + 0.03291208669543266, + 0.058432530611753464, + 0.05104740336537361, + -0.07065894454717636, + -0.03376605734229088, + -0.002328988164663315, + 0.014616183936595917, + -0.02626182697713375, + 0.046221405267715454, + 0.022209407761693, + -0.022042516618967056, + 0.00832273531705141, + -0.02817392908036709, + -0.022311683744192123, + -0.0012883107410743833, + 0.045586567372083664, + 0.036259911954402924, + 0.0021793527994304895, + 0.012211238034069538, + 0.009500940330326557, + 0.014388056471943855, + -0.006968256086111069, + 0.01394367590546608, + 0.03832881525158882, + -0.03186405077576637, + 0.00840744562447071, + -0.013643509708344936, + 0.007603461388498545, + 0.0033729849383234978, + 0.013005086220800877, + 0.024159906432032585, + -0.004308560397475958, + 0.032316654920578, + 0.00546879880130291, + 0.013974368572235107, + 0.06624308973550797, + 0.010526999831199646, + -0.04078470170497894, + -0.016591699793934822, + -0.003694466082379222, + -0.02095809578895569, + 0.029016772285103798, + 0.025617817416787148, + -0.010549499653279781, + 0.018108438700437546, + -0.00028947656392119825, + -0.042228806763887405, + 0.008322463370859623, + 0.020035985857248306, + -0.01081426814198494, + -0.014894822612404823, + 0.05130220577120781, + -0.010008169338107109, + -0.04114028811454773, + -0.0024317887146025896, + 0.03879571706056595, + -0.027351684868335724, + -0.029687397181987762, + 0.004803664516657591, + -0.006525125354528427, + -0.003111467696726322, + 0.009819800965487957, + 0.009924309328198433, + -0.07535731792449951, + 0.026950763538479805, + -0.04243321716785431, + 0.023490438237786293, + -0.008687988854944706, + -0.017979320138692856, + -0.008215818554162979, + -0.013817418366670609, + 0.023481370881199837, + 0.03727010637521744, + 0.008486934937536716, + 0.05204811319708824, + 0.018374331295490265, + -0.0217224583029747, + -0.019810568541288376, + -0.016522476449608803, + 0.06314573436975479, + -0.005394955165684223, + 0.028550174087285995, + 0.009453404694795609, + 0.03789595514535904, + 0.0014155276585370302, + 0.043733201920986176, + -0.00597009202465415, + 0.023255590349435806, + 0.03290838003158569, + 0.001254177070222795, + -0.03256387263536453, + -0.006253833882510662, + 0.06410900503396988, + 0.004187806975096464, + 0.025788236409425735, + -0.02977936901152134, + 0.015747565776109695, + -0.005101967602968216, + -0.018549447879195213, + 0.04633917659521103, + -0.0038384736981242895, + -0.009042670018970966, + 0.045430440455675125, + 0.015745969489216805, + 0.004589959047734737, + 0.006097143981605768, + 0.010901609435677528, + -0.06730777025222778, + -0.026084881275892258, + 0.003633950836956501, + -0.02018374390900135, + 0.028248965740203857, + 0.025721298530697823, + 0.05957205221056938, + 0.03806333243846893, + 0.04384124279022217, + 0.022846650332212448, + 0.07055970281362534, + 0.04863680899143219, + -0.0280617643147707, + -0.007124799769371748, + 0.007850217632949352, + -0.0015066524501889944, + -0.006744216661900282, + -0.014858445152640343, + -0.010598799213767052, + -0.06706836819648743, + -0.004910520277917385, + 0.029753463342785835, + -0.03185206651687622, + -0.0476350337266922, + -0.012068722397089005, + 0.04001805931329727, + -0.02616683766245842, + 0.032585419714450836, + -0.0063676415011286736, + 0.01699127070605755, + -0.009337509982287884, + -0.018481556326150894, + -0.016834305599331856, + -0.05032462999224663, + -0.0016227440210059285, + 0.006387660279870033, + 0.0021111038513481617, + 0.029920214787125587, + 0.018987925723195076, + 0.0058491104282438755, + 0.028183897957205772, + -0.0029647715855389833, + 0.006461241282522678, + -0.008717779070138931, + -0.014057681895792484, + -0.010591910220682621, + -0.021438555791974068, + 0.05116991698741913, + 0.03948652371764183, + 0.012003145180642605, + 0.05402052029967308, + -0.032920461148023605, + 0.011052411049604416, + 0.02017076127231121, + 0.05296896770596504, + -0.015856191515922546, + -0.02648812159895897, + 0.013666142709553242, + 0.01505154650658369, + -0.0034644489642232656, + 0.013673268258571625, + 0.022028522565960884, + 0.02551058866083622, + 0.018258098512887955, + 0.007342537399381399, + 0.01367469783872366, + -0.01365423109382391, + -0.0261926781386137, + -0.04282493516802788, + -0.012548241764307022, + -0.031617671251297, + -0.013877135701477528, + 0.0025892765261232853, + 0.034630537033081055, + 0.05838903412222862, + -0.01917693205177784, + -0.014046398922801018, + 0.000140307834954001, + -0.015239361673593521, + -0.005706230644136667, + -0.0026523303240537643, + 0.00804436206817627, + 0.022975869476795197, + -0.010957131162285805, + 0.0011564232409000397, + -0.00338548980653286, + -0.0508505143225193, + 0.030414516106247902, + -0.00906157772988081, + 0.009105802513659, + 0.027238845825195312, + -0.08628341555595398, + -0.008427499793469906, + -0.006338702980428934, + -0.017981693148612976, + -0.003787339199334383, + 0.01372926589101553, + -0.04929317906498909, + 0.01051567867398262, + 0.03434637933969498, + -0.008651690557599068, + 0.0029477926436811686, + -0.013656134717166424, + 0.02682396024465561, + 0.022849341854453087, + -0.012111956253647804, + 0.030370434746146202, + -0.06213803589344025, + -0.016147952526807785, + 0.08988979458808899, + -0.031755540519952774, + -0.005400430411100388, + 0.03365795314311981, + 0.0013827811926603317, + 0.04558301717042923, + -0.01944412663578987, + 0.047862663865089417, + -0.008816802874207497, + 0.0035925512202084064, + -0.06041926518082619, + -0.03558993712067604, + -0.018455395475029945, + 0.0112827243283391, + 0.046399205923080444, + -0.008579932153224945, + 0.030949249863624573, + 0.023780658841133118, + -0.028772728517651558, + 0.004042459186166525, + -0.009343907237052917, + 0.02148553729057312, + -0.04444408044219017, + 0.03086002729833126, + -0.012232224456965923, + 0.03052593022584915, + 0.01413752231746912, + -0.00363087747246027, + 0.009007961489260197, + 0.027095647528767586, + 0.036472171545028687, + 0.028931327164173126, + -0.04178299754858017, + 0.0007927287952043116, + -0.004962570033967495, + 0.010250034742057323, + 0.008548522368073463, + 0.021560465916991234, + 0.008888306096196175, + 0.03214035555720329, + 0.024108894169330597, + 0.01470613107085228, + -0.016194432973861694, + 0.0064420681446790695, + 0.04445295035839081, + -0.004213877022266388, + -0.005510952323675156, + -0.017451267689466476, + 0.03367631137371063, + 0.018276136368513107, + -0.012479925528168678, + 0.0032589719630777836, + -0.020025992766022682, + -0.009857109747827053, + -0.05180635303258896, + -0.008097291924059391, + -0.020081883296370506, + -0.01699046976864338, + 0.011628433130681515, + -0.006329878233373165, + -0.036662597209215164, + -0.022626450285315514, + -0.021029677242040634, + -0.0377374142408371, + -0.022792961448431015, + -0.006737920921295881, + 0.021406469866633415, + -0.055507369339466095, + -0.05102652311325073, + -0.0005814990727230906, + 0.01803787611424923, + -0.012722773477435112, + -0.018936092033982277, + 0.012126177549362183, + 0.011859937570989132, + -0.03201305866241455, + -0.024602307006716728, + -0.03842389956116676, + -0.0019073178991675377, + 0.05475657060742378, + -0.031966682523489, + -0.20830301940441132, + 0.012279191985726357, + 0.033405184745788574, + -0.025119803845882416, + 0.02976243384182453, + -0.023737043142318726, + -0.0426027849316597, + -0.024789312854409218, + 0.08287238329648972, + -0.012735029682517052, + -0.02916696108877659, + -0.00483438977971673, + -0.05310559272766113, + -0.007113052532076836, + 0.04755304753780365, + -0.0008659433806315064, + -0.006125824525952339, + 0.0337761715054512, + 0.02006487362086773, + -0.008797365240752697, + -0.04096890613436699, + -0.018489748239517212, + 0.03374563902616501, + 0.010460725985467434, + 0.0462099127471447, + -0.06521918624639511, + -0.008828939869999886, + 0.0010993658797815442, + 0.03617260605096817, + -0.007756139617413282, + -0.031840018928050995, + -0.02730613201856613, + -0.02897568792104721, + 0.00180403096601367, + 0.027762500569224358, + -0.0014431070303544402, + -0.02256607450544834, + 0.00026335156871937215, + 0.0599030926823616, + -0.0028933624271303415, + -0.022167986258864403, + 0.011483773589134216, + -0.014741613529622555, + -0.033078502863645554, + -0.04396243393421173, + -0.006488406099379063, + 0.027313947677612305, + -0.05013969540596008, + -0.046316295862197876, + -0.048080604523420334, + -0.02174241654574871, + 0.002347409725189209, + -0.03159633278846741, + -0.01286842580884695, + 0.027270538732409477, + -0.04421807825565338, + -0.023035183548927307, + 0.009107952006161213, + -0.007931916043162346, + 0.06298748403787613, + -0.010076748207211494, + -0.04212477430701256, + -0.06444337964057922, + 0.017816929146647453, + 0.008650356903672218, + -0.008252098225057125, + 0.03915150463581085, + -0.006211346480995417, + -0.029086243361234665, + 0.02539663575589657, + 0.018931930884718895, + 0.007681270595639944, + -0.045202482491731644, + 0.018191026523709297, + -0.04999200254678726, + 0.0019298805855214596, + 0.03213529661297798, + -0.009509213268756866, + -0.012883023358881474, + 0.0018169092945754528, + -0.03838331252336502, + 0.007685525342822075, + -0.030290678143501282, + -0.020784694701433182, + -0.004612106829881668, + 0.037329625338315964, + -0.03761892020702362, + 0.006507837679237127, + -0.013387695886194706, + -0.029673071578145027, + 0.023734647780656815, + -0.035589028149843216, + 0.03293817862868309, + 0.00018127942166756839, + 0.013632168062031269, + -0.04605011269450188, + 0.001839012373238802, + 0.08389141410589218, + -0.021381760016083717, + 0.0053498148918151855, + 0.0420515313744545, + 0.050614017993211746, + -0.030694706365466118, + 0.005227542482316494, + -0.003097192384302616, + -0.019282478839159012, + -0.009066129103302956, + 0.032834507524967194, + 0.0392480194568634, + -0.04873361811041832, + -0.02524818293750286, + -0.0036731662694364786, + -0.015791267156600952, + 0.02608574368059635, + -0.010344231501221657, + 0.035860493779182434, + -0.01692425087094307, + -0.010126749984920025, + 0.004166270140558481, + -0.020157892256975174, + -0.06899076700210571, + 0.003458562307059765, + 0.06725051999092102, + -0.08162353932857513, + 0.01825786381959915, + -0.016596291214227676, + -0.02741207554936409, + 0.0021401301492005587, + 0.015485628508031368, + -0.004590863827615976, + -0.01754390448331833, + -0.060239020735025406, + -0.029201781377196312, + 0.020242372527718544, + -0.005814275238662958, + -0.04401073604822159, + -0.048212986439466476, + 0.025246409699320793, + 0.009146126918494701, + 0.016770247370004654, + 0.044476721435785294, + 0.033011626452207565, + 0.018314186483621597, + 0.025626717135310173, + 0.027040621265769005, + 0.04458078369498253, + -0.04554470255970955, + -0.046757765114307404, + 0.00499960221350193, + -0.001518383389338851, + -0.0010033997241407633, + -0.007891355082392693, + -0.011428908444941044, + 0.0038406497333198786, + 0.018944026902318, + -0.014704111032187939, + 0.0038604368455708027, + -0.04271119087934494, + -0.036563243716955185, + -0.03623231127858162, + -0.04191727936267853, + 0.0011887417640537024, + 0.051631662994623184, + 0.016292477026581764, + -0.00515693100169301, + 0.021923739463090897, + -0.007323189172893763, + 0.0034489671234041452, + 0.021977417171001434, + 0.025491874665021896, + 0.02655629813671112, + 0.03194888308644295, + 0.0023770860861986876, + -0.019877245649695396, + 0.011530621908605099, + -0.007378903217613697, + 0.023632757365703583, + 0.07805845886468887, + -0.06732141226530075, + -0.04045461490750313, + -0.020166844129562378, + -0.015956057235598564, + -0.007738949730992317, + -0.02221641130745411, + -0.00011409397120587528, + -0.031704556196928024, + 0.006996236741542816, + -0.017422057688236237, + 0.0077502429485321045, + 0.06482279300689697, + -0.03198590129613876, + -0.014389376156032085, + 0.05045264959335327, + 0.017990874126553535, + -0.0062853265553712845, + -0.021538320928812027, + 0.02219160459935665, + 0.016452554613351822, + -0.016343727707862854, + 0.019444739446043968, + -0.012018387205898762, + -0.03474026545882225, + -0.04465792700648308, + -0.03613681346178055, + -0.04903527349233627, + 0.017347920686006546, + -0.025677626952528954, + -0.01066385954618454, + 0.0015619783662259579, + 0.017323894426226616, + 0.00646140705794096, + 0.026005659252405167, + -0.03258197009563446, + 0.06512430310249329, + 0.021463433280587196, + 0.023544099181890488, + -0.011928308755159378, + 0.026693787425756454, + -0.04043189808726311, + -0.016095992177724838, + 0.01213125605136156, + -0.029467647895216942, + -0.003772783325985074, + 0.004925335291773081, + -0.030109306797385216, + -0.017491687089204788, + 0.021796943619847298, + 0.03961903974413872, + 0.025019735097885132, + 0.009634989313781261, + 0.008004709146916866, + 0.008586048148572445, + 0.028207149356603622, + 0.020454756915569305, + -0.015108142979443073, + 0.037390634417533875, + 0.049543414264917374, + 0.04195686802268028, + 0.03264535218477249, + -0.06146916747093201, + 0.05023035779595375, + 0.0033935722894966602, + 0.0381556898355484, + -0.01711106300354004, + -0.009851740673184395, + -0.03739236295223236, + 0.04077133163809776, + -0.036508142948150635, + 0.030934127047657967, + -0.057432979345321655, + 0.01713881827890873, + 0.012707455083727837, + -0.01326854806393385, + 0.03284291550517082, + -0.05934964120388031, + -0.004297229927033186, + 0.04701299965381622, + -0.04109788313508034, + 0.009807238355278969, + 0.01876932568848133, + 0.03675206005573273, + -0.023492969572544098, + 0.04140949621796608, + 0.037333644926548004, + -0.04521091654896736, + -0.044086746871471405, + -0.01617482118308544, + -0.011979220435023308, + -0.00980219803750515, + 0.008054448291659355, + -0.018051551654934883, + 0.01857340708374977, + 0.028104456141591072, + 0.02108258195221424, + 0.021605802699923515, + 0.004357744939625263, + -0.023117326200008392, + -0.013717963360249996, + -0.0005997371044941247, + 0.019163908436894417, + -0.05397503450512886, + 0.004755691159516573, + -0.022543836385011673, + 0.022406205534934998, + -0.03557561710476875, + -0.023239079862833023, + 0.015066863037645817, + 0.010571423918008804, + -0.01830492541193962, + 0.02738388627767563, + 0.0015923595055937767, + 0.011423752643167973, + 0.029959186911582947, + -0.028333624824881554, + 0.02825409360229969, + -0.0033188818488270044, + -0.020686671137809753, + -0.030213572084903717, + 0.0045396857894957066, + -0.002853690180927515, + -0.010961533524096012, + -0.030884260311722755, + 0.021235954016447067, + 0.029801206663250923, + -0.0017682340694591403, + -0.021077800542116165, + -0.011756698600947857, + -0.01143163163214922, + -0.03345923498272896, + -0.03428651764988899, + 0.05735395848751068, + 0.013249286450445652, + -0.03272906318306923, + 0.0043847281485795975, + -0.04382368549704552, + -0.03494689613580704, + 0.0031293844804167747, + -0.028288982808589935, + 0.04488981515169144, + 0.023047233000397682, + 0.001025717006996274, + 0.010647145099937916, + -0.005209266673773527, + 0.0050440458580851555, + -0.03157447651028633, + 0.018987910822033882, + -0.01106954738497734, + 0.020170150324702263, + 0.01725236512720585, + 0.03856505826115608, + -0.015215910039842129, + -0.05163946747779846, + -0.052423447370529175, + -0.039678726345300674, + -0.00018781247490551323 + ], + "attn_peak_rel": -2.875, + "attn_entropy": 0.12596949050202966, + "attn_spread_pm2": 0.1469257604330778, + "attn_role": "looks_back", + "n_pairs": 8 + } + } +} \ No newline at end of file diff --git a/data/e1_cross/EleutherAI--pythia-1b_seed42.json b/data/e1_cross/EleutherAI--pythia-1b_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..d7826cdd9004bd2f8592f865342972016e8ee2f0 --- /dev/null +++ b/data/e1_cross/EleutherAI--pythia-1b_seed42.json @@ -0,0 +1,109 @@ +{ + "model": "EleutherAI/pythia-1b", + "n_layers": 16, + "L_crit": 15, + "n_prompts": 30, + "seed": 42, + "logp_gap": 1.96565416653951, + "mean_delta_pre_lcrit": 42.40066298224842, + "cross_wins_layers": 14, + "interpretation": "Crystallized residual IS useful when injected early", + "per_layer": [ + { + "layer": 0, + "R_std": 0.0, + "R_cross": 58.92, + "delta": 58.92 + }, + { + "layer": 1, + "R_std": 8.25, + "R_cross": 48.52, + "delta": 40.27 + }, + { + "layer": 2, + "R_std": 5.5, + "R_cross": 56.91, + "delta": 51.41 + }, + { + "layer": 3, + "R_std": -1.27, + "R_cross": 62.16, + "delta": 63.43 + }, + { + "layer": 4, + "R_std": -2.09, + "R_cross": 65.77, + "delta": 67.86 + }, + { + "layer": 5, + "R_std": 2.64, + "R_cross": 64.96, + "delta": 62.32 + }, + { + "layer": 6, + "R_std": 8.85, + "R_cross": 65.87, + "delta": 57.02 + }, + { + "layer": 7, + "R_std": 9.39, + "R_cross": 65.24, + "delta": 55.86 + }, + { + "layer": 8, + "R_std": 23.36, + "R_cross": 63.94, + "delta": 40.58 + }, + { + "layer": 9, + "R_std": 25.04, + "R_cross": 66.4, + "delta": 41.36 + }, + { + "layer": 10, + "R_std": 31.25, + "R_cross": 64.8, + "delta": 33.54 + }, + { + "layer": 11, + "R_std": 32.09, + "R_cross": 66.88, + "delta": 34.79 + }, + { + "layer": 12, + "R_std": 51.02, + "R_cross": 68.28, + "delta": 17.26 + }, + { + "layer": 13, + "R_std": 69.72, + "R_cross": 72.96, + "delta": 3.23 + }, + { + "layer": 14, + "R_std": 73.34, + "R_cross": 81.51, + "delta": 8.16 + }, + { + "layer": 15, + "R_std": 92.53, + "R_cross": 92.53, + "delta": 0.0 + } + ] +} \ No newline at end of file diff --git a/data/e1_cross/EleutherAI--pythia-410m_seed42.json b/data/e1_cross/EleutherAI--pythia-410m_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..97bfbae2289d00b88ba7092db0ab8d5268bba05f --- /dev/null +++ b/data/e1_cross/EleutherAI--pythia-410m_seed42.json @@ -0,0 +1,157 @@ +{ + "model": "EleutherAI/pythia-410m", + "n_layers": 24, + "L_crit": 23, + "n_prompts": 30, + "seed": 42, + "logp_gap": -0.152927112579345, + "mean_delta_pre_lcrit": 259.4164504529749, + "cross_wins_layers": 21, + "interpretation": "Crystallized residual IS useful when injected early", + "per_layer": [ + { + "layer": 0, + "R_std": -0.0, + "R_cross": 171.71, + "delta": 171.71 + }, + { + "layer": 1, + "R_std": -660.87, + "R_cross": 266.81, + "delta": 927.68 + }, + { + "layer": 2, + "R_std": -658.85, + "R_cross": 220.01, + "delta": 878.85 + }, + { + "layer": 3, + "R_std": 128.4, + "R_cross": 139.75, + "delta": 11.35 + }, + { + "layer": 4, + "R_std": 22.46, + "R_cross": 176.42, + "delta": 153.96 + }, + { + "layer": 5, + "R_std": 42.35, + "R_cross": 223.68, + "delta": 181.33 + }, + { + "layer": 6, + "R_std": 107.88, + "R_cross": 128.35, + "delta": 20.46 + }, + { + "layer": 7, + "R_std": 181.5, + "R_cross": 258.71, + "delta": 77.22 + }, + { + "layer": 8, + "R_std": -194.83, + "R_cross": 130.71, + "delta": 325.54 + }, + { + "layer": 9, + "R_std": 1.17, + "R_cross": 289.67, + "delta": 288.5 + }, + { + "layer": 10, + "R_std": -296.2, + "R_cross": 300.2, + "delta": 596.4 + }, + { + "layer": 11, + "R_std": -31.83, + "R_cross": 343.88, + "delta": 375.71 + }, + { + "layer": 12, + "R_std": -133.52, + "R_cross": 242.76, + "delta": 376.27 + }, + { + "layer": 13, + "R_std": -160.11, + "R_cross": 324.7, + "delta": 484.8 + }, + { + "layer": 14, + "R_std": -238.56, + "R_cross": 180.94, + "delta": 419.5 + }, + { + "layer": 15, + "R_std": -246.76, + "R_cross": 68.93, + "delta": 315.69 + }, + { + "layer": 16, + "R_std": -44.37, + "R_cross": 66.65, + "delta": 111.01 + }, + { + "layer": 17, + "R_std": 18.56, + "R_cross": 186.4, + "delta": 167.84 + }, + { + "layer": 18, + "R_std": -59.69, + "R_cross": 53.83, + "delta": 113.52 + }, + { + "layer": 19, + "R_std": -166.8, + "R_cross": 13.81, + "delta": 180.61 + }, + { + "layer": 20, + "R_std": -35.21, + "R_cross": 60.2, + "delta": 95.41 + }, + { + "layer": 21, + "R_std": 194.75, + "R_cross": 96.04, + "delta": -98.71 + }, + { + "layer": 22, + "R_std": 208.52, + "R_cross": 0.43, + "delta": -208.09 + }, + { + "layer": 23, + "R_std": 113.88, + "R_cross": 113.88, + "delta": 0.0 + } + ] +} \ No newline at end of file diff --git a/data/e1_cross/EleutherAI--pythia-70m_seed42.json b/data/e1_cross/EleutherAI--pythia-70m_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..00bc2dc984821bee973106104a79539e7abcc1b9 --- /dev/null +++ b/data/e1_cross/EleutherAI--pythia-70m_seed42.json @@ -0,0 +1,49 @@ +{ + "model": "EleutherAI/pythia-70m", + "n_layers": 6, + "L_crit": 4, + "n_prompts": 30, + "seed": 42, + "logp_gap": 0.6075656890869148, + "mean_delta_pre_lcrit": -116.41531740070917, + "cross_wins_layers": 0, + "interpretation": "Type mismatch: L_crit residual corrupts early layers", + "per_layer": [ + { + "layer": 0, + "R_std": 0.0, + "R_cross": -59.78, + "delta": -59.78 + }, + { + "layer": 1, + "R_std": -60.06, + "R_cross": -95.11, + "delta": -35.05 + }, + { + "layer": 2, + "R_std": -25.53, + "R_cross": -183.66, + "delta": -158.13 + }, + { + "layer": 3, + "R_std": -102.32, + "R_cross": -315.01, + "delta": -212.7 + }, + { + "layer": 4, + "R_std": 123.53, + "R_cross": 123.53, + "delta": 0.0 + }, + { + "layer": 5, + "R_std": 98.83, + "R_cross": 90.63, + "delta": -8.2 + } + ] +} \ No newline at end of file diff --git a/data/e1_cross/gpt2-medium_seed42.json b/data/e1_cross/gpt2-medium_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..71d112afb73b9944a74b9beb11c992635d41ed19 --- /dev/null +++ b/data/e1_cross/gpt2-medium_seed42.json @@ -0,0 +1,157 @@ +{ + "model": "gpt2-medium", + "n_layers": 24, + "L_crit": 23, + "n_prompts": 30, + "seed": 42, + "logp_gap": 4.162486102183659, + "mean_delta_pre_lcrit": 32.67454236379804, + "cross_wins_layers": 22, + "interpretation": "Crystallized residual IS useful when injected early", + "per_layer": [ + { + "layer": 0, + "R_std": 5.79, + "R_cross": 17.04, + "delta": 11.25 + }, + { + "layer": 1, + "R_std": -1.54, + "R_cross": 20.2, + "delta": 21.74 + }, + { + "layer": 2, + "R_std": -21.26, + "R_cross": 19.01, + "delta": 40.28 + }, + { + "layer": 3, + "R_std": -29.6, + "R_cross": 21.67, + "delta": 51.27 + }, + { + "layer": 4, + "R_std": -26.49, + "R_cross": 18.44, + "delta": 44.93 + }, + { + "layer": 5, + "R_std": -25.21, + "R_cross": 29.79, + "delta": 55.0 + }, + { + "layer": 6, + "R_std": -19.6, + "R_cross": 28.69, + "delta": 48.29 + }, + { + "layer": 7, + "R_std": -8.72, + "R_cross": 36.14, + "delta": 44.87 + }, + { + "layer": 8, + "R_std": -2.99, + "R_cross": 38.69, + "delta": 41.68 + }, + { + "layer": 9, + "R_std": 1.06, + "R_cross": 42.66, + "delta": 41.61 + }, + { + "layer": 10, + "R_std": 3.4, + "R_cross": 44.52, + "delta": 41.13 + }, + { + "layer": 11, + "R_std": 17.14, + "R_cross": 46.96, + "delta": 29.82 + }, + { + "layer": 12, + "R_std": 28.7, + "R_cross": 49.77, + "delta": 21.07 + }, + { + "layer": 13, + "R_std": 25.42, + "R_cross": 52.57, + "delta": 27.15 + }, + { + "layer": 14, + "R_std": 26.2, + "R_cross": 55.31, + "delta": 29.11 + }, + { + "layer": 15, + "R_std": 23.99, + "R_cross": 58.04, + "delta": 34.06 + }, + { + "layer": 16, + "R_std": 24.36, + "R_cross": 61.94, + "delta": 37.59 + }, + { + "layer": 17, + "R_std": 30.28, + "R_cross": 66.59, + "delta": 36.31 + }, + { + "layer": 18, + "R_std": 33.87, + "R_cross": 67.98, + "delta": 34.11 + }, + { + "layer": 19, + "R_std": 42.27, + "R_cross": 69.1, + "delta": 26.83 + }, + { + "layer": 20, + "R_std": 53.52, + "R_cross": 73.36, + "delta": 19.84 + }, + { + "layer": 21, + "R_std": 68.11, + "R_cross": 77.5, + "delta": 9.38 + }, + { + "layer": 22, + "R_std": 83.15, + "R_cross": 87.35, + "delta": 4.2 + }, + { + "layer": 23, + "R_std": 93.07, + "R_cross": 93.07, + "delta": 0.0 + } + ] +} \ No newline at end of file diff --git a/data/e1_h3/BlinkDL--rwkv-4-world-3b_seed42.json b/data/e1_h3/BlinkDL--rwkv-4-world-3b_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..c900011c22d7bb8f3cea21c5548902d36ac42843 --- /dev/null +++ b/data/e1_h3/BlinkDL--rwkv-4-world-3b_seed42.json @@ -0,0 +1,188 @@ +{ + "model_type": "rwkv_rnn", + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 2, + "dist_short": 10, + "dist_long": 1000, + "logp_gap": 5.3643, + "ref_logp_d10": { + "mean": -8.0806, + "std": 2.5551 + }, + "baseline_logp_d1000": { + "mean": -13.4449, + "std": 0.5371 + }, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -13.4449, + "logp_transplant_std": 0.5371 + }, + "1": { + "recovery_pct": 9.32, + "logp_transplant_mean": -12.9449, + "logp_transplant_std": 0.6751 + }, + "2": { + "recovery_pct": 6.6, + "logp_transplant_mean": -13.0907, + "logp_transplant_std": 0.7562 + }, + "3": { + "recovery_pct": 2.42, + "logp_transplant_mean": -13.3152, + "logp_transplant_std": 0.7399 + }, + "4": { + "recovery_pct": -3.9, + "logp_transplant_mean": -13.654, + "logp_transplant_std": 0.5254 + }, + "5": { + "recovery_pct": -5.81, + "logp_transplant_mean": -13.7564, + "logp_transplant_std": 0.3247 + }, + "6": { + "recovery_pct": -11.03, + "logp_transplant_mean": -14.0365, + "logp_transplant_std": 0.535 + }, + "7": { + "recovery_pct": -12.82, + "logp_transplant_mean": -14.1328, + "logp_transplant_std": 0.8057 + }, + "8": { + "recovery_pct": -10.49, + "logp_transplant_mean": -14.0078, + "logp_transplant_std": 0.716 + }, + "9": { + "recovery_pct": -9.73, + "logp_transplant_mean": -13.9669, + "logp_transplant_std": 0.7431 + }, + "10": { + "recovery_pct": -8.57, + "logp_transplant_mean": -13.9048, + "logp_transplant_std": 0.731 + }, + "11": { + "recovery_pct": -9.3, + "logp_transplant_mean": -13.944, + "logp_transplant_std": 0.7777 + }, + "12": { + "recovery_pct": -6.77, + "logp_transplant_mean": -13.8083, + "logp_transplant_std": 0.7119 + }, + "13": { + "recovery_pct": -7.08, + "logp_transplant_mean": -13.8246, + "logp_transplant_std": 0.782 + }, + "14": { + "recovery_pct": -10.01, + "logp_transplant_mean": -13.9817, + "logp_transplant_std": 0.7615 + }, + "15": { + "recovery_pct": -3.46, + "logp_transplant_mean": -13.6305, + "logp_transplant_std": 0.5568 + }, + "16": { + "recovery_pct": -2.56, + "logp_transplant_mean": -13.5825, + "logp_transplant_std": 0.3059 + }, + "17": { + "recovery_pct": 7.37, + "logp_transplant_mean": -13.0499, + "logp_transplant_std": 0.231 + }, + "18": { + "recovery_pct": 13.75, + "logp_transplant_mean": -12.7073, + "logp_transplant_std": 0.2173 + }, + "19": { + "recovery_pct": 18.04, + "logp_transplant_mean": -12.4773, + "logp_transplant_std": 0.6731 + }, + "20": { + "recovery_pct": 24.66, + "logp_transplant_mean": -12.1223, + "logp_transplant_std": 0.4863 + }, + "21": { + "recovery_pct": 23.65, + "logp_transplant_mean": -12.1762, + "logp_transplant_std": 0.5121 + }, + "22": { + "recovery_pct": 36.9, + "logp_transplant_mean": -11.4657, + "logp_transplant_std": 0.1153 + }, + "23": { + "recovery_pct": 34.61, + "logp_transplant_mean": -11.5886, + "logp_transplant_std": 0.0028 + }, + "24": { + "recovery_pct": 43.64, + "logp_transplant_mean": -11.104, + "logp_transplant_std": 0.4192 + }, + "25": { + "recovery_pct": 60.76, + "logp_transplant_mean": -10.1854, + "logp_transplant_std": 0.2813 + }, + "26": { + "recovery_pct": 65.54, + "logp_transplant_mean": -9.9294, + "logp_transplant_std": 0.8151 + }, + "27": { + "recovery_pct": 70.47, + "logp_transplant_mean": -9.6648, + "logp_transplant_std": 1.0024 + }, + "28": { + "recovery_pct": 73.63, + "logp_transplant_mean": -9.4952, + "logp_transplant_std": 1.2598 + }, + "29": { + "recovery_pct": 75.04, + "logp_transplant_mean": -9.4197, + "logp_transplant_std": 1.3715 + }, + "30": { + "recovery_pct": 80.81, + "logp_transplant_mean": -9.1098, + "logp_transplant_std": 1.5623 + }, + "31": { + "recovery_pct": 99.03, + "logp_transplant_mean": -8.1328, + "logp_transplant_std": 2.5247 + } + }, + "L_crit_90": 31, + "L_crit_99": 31, + "alpha_90": 0.9688, + "alpha_99": 0.9688, + "n_layers": 32, + "d_model": 2560, + "seed": 42, + "model": "BlinkDL/rwkv-4-world-3b", + "runtime_seconds": 3226.7 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-160m_seed123.json b/data/e1_h3/EleutherAI--pythia-160m_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..43a0fddf903ec22e230bbdb604c4073995b5ce06 --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-160m_seed123.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -17.5457, + "std": 4.8227 + }, + "baseline_logp_d1000": { + "mean": -17.857, + "std": 5.8249 + }, + "logp_gap": 0.3112, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -17.857, + "logp_transplant_std": 5.8249 + }, + "1": { + "recovery_pct": -94.41, + "logp_transplant_mean": -18.1508, + "logp_transplant_std": 5.4495 + }, + "2": { + "recovery_pct": -186.81, + "logp_transplant_mean": -18.4384, + "logp_transplant_std": 6.3528 + }, + "3": { + "recovery_pct": 123.43, + "logp_transplant_mean": -17.4728, + "logp_transplant_std": 4.5409 + }, + "4": { + "recovery_pct": 147.9, + "logp_transplant_mean": -17.3966, + "logp_transplant_std": 4.3101 + }, + "5": { + "recovery_pct": -20.47, + "logp_transplant_mean": -17.9206, + "logp_transplant_std": 4.66 + }, + "6": { + "recovery_pct": -236.04, + "logp_transplant_mean": -18.5916, + "logp_transplant_std": 5.54 + }, + "7": { + "recovery_pct": -2.99, + "logp_transplant_mean": -17.8663, + "logp_transplant_std": 4.4843 + }, + "8": { + "recovery_pct": 27.14, + "logp_transplant_mean": -17.7725, + "logp_transplant_std": 4.6484 + }, + "9": { + "recovery_pct": 108.07, + "logp_transplant_mean": -17.5206, + "logp_transplant_std": 4.8187 + }, + "10": { + "recovery_pct": 132.76, + "logp_transplant_mean": -17.4438, + "logp_transplant_std": 4.6504 + }, + "11": { + "recovery_pct": 128.49, + "logp_transplant_mean": -17.457, + "logp_transplant_std": 4.7403 + } + }, + "L_crit_90": 3, + "L_crit_99": 3, + "alpha_90": 0.25, + "alpha_99": 0.25, + "recovery_at_Lcrit": 1.2343, + "n_layers": 12, + "d_model": 768, + "model": "EleutherAI/pythia-160m", + "seed": 123, + "runtime_seconds": 32.3 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-160m_seed42.json b/data/e1_h3/EleutherAI--pythia-160m_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..13a055aef58a5afeb08016ebcbbd301023bf7c44 --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-160m_seed42.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -22.4039, + "std": 32.5607 + }, + "baseline_logp_d1000": { + "mean": -22.5344, + "std": 32.3306 + }, + "logp_gap": 0.1304, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -22.5344, + "logp_transplant_std": 32.3306 + }, + "1": { + "recovery_pct": -345.29, + "logp_transplant_mean": -22.9848, + "logp_transplant_std": 32.4531 + }, + "2": { + "recovery_pct": -350.06, + "logp_transplant_mean": -22.991, + "logp_transplant_std": 32.2325 + }, + "3": { + "recovery_pct": -282.43, + "logp_transplant_mean": -22.9028, + "logp_transplant_std": 32.604 + }, + "4": { + "recovery_pct": 408.6, + "logp_transplant_mean": -22.0014, + "logp_transplant_std": 32.8209 + }, + "5": { + "recovery_pct": -126.56, + "logp_transplant_mean": -22.6995, + "logp_transplant_std": 31.0138 + }, + "6": { + "recovery_pct": 313.89, + "logp_transplant_mean": -22.125, + "logp_transplant_std": 30.7608 + }, + "7": { + "recovery_pct": 244.26, + "logp_transplant_mean": -22.2158, + "logp_transplant_std": 32.1041 + }, + "8": { + "recovery_pct": 402.51, + "logp_transplant_mean": -22.0094, + "logp_transplant_std": 32.2605 + }, + "9": { + "recovery_pct": 184.67, + "logp_transplant_mean": -22.2935, + "logp_transplant_std": 32.8244 + }, + "10": { + "recovery_pct": 142.87, + "logp_transplant_mean": -22.348, + "logp_transplant_std": 32.5819 + }, + "11": { + "recovery_pct": 81.72, + "logp_transplant_mean": -22.4278, + "logp_transplant_std": 32.6702 + } + }, + "L_crit_90": 4, + "L_crit_99": 4, + "alpha_90": 0.3333, + "alpha_99": 0.3333, + "recovery_at_Lcrit": 4.086, + "n_layers": 12, + "d_model": 768, + "model": "EleutherAI/pythia-160m", + "seed": 42, + "runtime_seconds": 32.5 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-160m_seed7.json b/data/e1_h3/EleutherAI--pythia-160m_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..9fc478fd5f4258e22385cb4feea69d57cfb8fe71 --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-160m_seed7.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -61.4318, + "std": 136.2948 + }, + "baseline_logp_d1000": { + "mean": -59.9124, + "std": 135.2336 + }, + "logp_gap": -1.5194, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -59.9124, + "logp_transplant_std": 135.2336 + }, + "1": { + "recovery_pct": 2.47, + "logp_transplant_mean": -59.9499, + "logp_transplant_std": 135.1706 + }, + "2": { + "recovery_pct": 44.57, + "logp_transplant_mean": -60.5896, + "logp_transplant_std": 135.8501 + }, + "3": { + "recovery_pct": 21.15, + "logp_transplant_mean": -60.2337, + "logp_transplant_std": 135.8855 + }, + "4": { + "recovery_pct": 35.23, + "logp_transplant_mean": -60.4477, + "logp_transplant_std": 136.3003 + }, + "5": { + "recovery_pct": 87.11, + "logp_transplant_mean": -61.2361, + "logp_transplant_std": 136.6496 + }, + "6": { + "recovery_pct": 103.73, + "logp_transplant_mean": -61.4885, + "logp_transplant_std": 136.5528 + }, + "7": { + "recovery_pct": 77.84, + "logp_transplant_mean": -61.0951, + "logp_transplant_std": 136.0754 + }, + "8": { + "recovery_pct": 74.94, + "logp_transplant_mean": -61.0511, + "logp_transplant_std": 136.1683 + }, + "9": { + "recovery_pct": 63.85, + "logp_transplant_mean": -60.8826, + "logp_transplant_std": 135.7116 + }, + "10": { + "recovery_pct": 89.21, + "logp_transplant_mean": -61.2679, + "logp_transplant_std": 136.3431 + }, + "11": { + "recovery_pct": 86.05, + "logp_transplant_mean": -61.2199, + "logp_transplant_std": 136.4501 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 12, + "d_model": 768, + "model": "EleutherAI/pythia-160m", + "seed": 7, + "runtime_seconds": 32.1 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-1b_seed123.json b/data/e1_h3/EleutherAI--pythia-1b_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..0bb0d5c6aae749800a863c79041491e0a143c5fe --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-1b_seed123.json @@ -0,0 +1,111 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -10.3046, + "std": 2.2192 + }, + "baseline_logp_d1000": { + "mean": -12.4779, + "std": 1.455 + }, + "logp_gap": 2.1733, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -12.4779, + "logp_transplant_std": 1.455 + }, + "1": { + "recovery_pct": 8.47, + "logp_transplant_mean": -12.2939, + "logp_transplant_std": 1.4295 + }, + "2": { + "recovery_pct": 8.85, + "logp_transplant_mean": -12.2857, + "logp_transplant_std": 1.4064 + }, + "3": { + "recovery_pct": 2.85, + "logp_transplant_mean": -12.416, + "logp_transplant_std": 1.4091 + }, + "4": { + "recovery_pct": 1.74, + "logp_transplant_mean": -12.4402, + "logp_transplant_std": 1.4058 + }, + "5": { + "recovery_pct": 5.2, + "logp_transplant_mean": -12.3649, + "logp_transplant_std": 1.3747 + }, + "6": { + "recovery_pct": 9.54, + "logp_transplant_mean": -12.2705, + "logp_transplant_std": 1.3938 + }, + "7": { + "recovery_pct": 11.2, + "logp_transplant_mean": -12.2345, + "logp_transplant_std": 1.3744 + }, + "8": { + "recovery_pct": 28.36, + "logp_transplant_mean": -11.8617, + "logp_transplant_std": 1.4408 + }, + "9": { + "recovery_pct": 32.62, + "logp_transplant_mean": -11.7691, + "logp_transplant_std": 1.4429 + }, + "10": { + "recovery_pct": 44.53, + "logp_transplant_mean": -11.5102, + "logp_transplant_std": 1.574 + }, + "11": { + "recovery_pct": 47.59, + "logp_transplant_mean": -11.4437, + "logp_transplant_std": 1.6022 + }, + "12": { + "recovery_pct": 67.9, + "logp_transplant_mean": -11.0023, + "logp_transplant_std": 1.7979 + }, + "13": { + "recovery_pct": 79.55, + "logp_transplant_mean": -10.7491, + "logp_transplant_std": 1.8821 + }, + "14": { + "recovery_pct": 79.98, + "logp_transplant_mean": -10.7398, + "logp_transplant_std": 2.0404 + }, + "15": { + "recovery_pct": 92.74, + "logp_transplant_mean": -10.4625, + "logp_transplant_std": 2.1161 + } + }, + "L_crit_90": 15, + "L_crit_99": null, + "alpha_90": 0.9375, + "alpha_99": null, + "recovery_at_Lcrit": 0.9274, + "n_layers": 16, + "d_model": 2048, + "model": "EleutherAI/pythia-1b", + "seed": 123, + "runtime_seconds": 162.9 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-1b_seed42.json b/data/e1_h3/EleutherAI--pythia-1b_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..75c36e28a50544b2600b057a2b4d3635484a735a --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-1b_seed42.json @@ -0,0 +1,111 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -10.3179, + "std": 1.5117 + }, + "baseline_logp_d1000": { + "mean": -12.4879, + "std": 1.5237 + }, + "logp_gap": 2.1701, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -12.4879, + "logp_transplant_std": 1.5237 + }, + "1": { + "recovery_pct": 5.48, + "logp_transplant_mean": -12.3691, + "logp_transplant_std": 1.3822 + }, + "2": { + "recovery_pct": 4.79, + "logp_transplant_mean": -12.384, + "logp_transplant_std": 1.3734 + }, + "3": { + "recovery_pct": -0.83, + "logp_transplant_mean": -12.5059, + "logp_transplant_std": 1.557 + }, + "4": { + "recovery_pct": -1.03, + "logp_transplant_mean": -12.5103, + "logp_transplant_std": 1.53 + }, + "5": { + "recovery_pct": 2.32, + "logp_transplant_mean": -12.4376, + "logp_transplant_std": 1.5829 + }, + "6": { + "recovery_pct": 8.31, + "logp_transplant_mean": -12.3076, + "logp_transplant_std": 1.5737 + }, + "7": { + "recovery_pct": 7.81, + "logp_transplant_mean": -12.3184, + "logp_transplant_std": 1.585 + }, + "8": { + "recovery_pct": 23.41, + "logp_transplant_mean": -11.98, + "logp_transplant_std": 1.6422 + }, + "9": { + "recovery_pct": 25.19, + "logp_transplant_mean": -11.9414, + "logp_transplant_std": 1.6004 + }, + "10": { + "recovery_pct": 32.63, + "logp_transplant_mean": -11.7798, + "logp_transplant_std": 1.5883 + }, + "11": { + "recovery_pct": 34.44, + "logp_transplant_mean": -11.7406, + "logp_transplant_std": 1.5632 + }, + "12": { + "recovery_pct": 54.39, + "logp_transplant_mean": -11.3077, + "logp_transplant_std": 1.5193 + }, + "13": { + "recovery_pct": 69.36, + "logp_transplant_mean": -10.9828, + "logp_transplant_std": 1.5043 + }, + "14": { + "recovery_pct": 74.57, + "logp_transplant_mean": -10.8697, + "logp_transplant_std": 1.4552 + }, + "15": { + "recovery_pct": 90.76, + "logp_transplant_mean": -10.5183, + "logp_transplant_std": 1.4555 + } + }, + "L_crit_90": 15, + "L_crit_99": null, + "alpha_90": 0.9375, + "alpha_99": null, + "recovery_at_Lcrit": 0.9076, + "n_layers": 16, + "d_model": 2048, + "model": "EleutherAI/pythia-1b", + "seed": 42, + "runtime_seconds": 163.0 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-1b_seed7.json b/data/e1_h3/EleutherAI--pythia-1b_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..93c8837da94743791e8089e4e370d92f6c5532de --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-1b_seed7.json @@ -0,0 +1,111 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -10.6954, + "std": 2.2961 + }, + "baseline_logp_d1000": { + "mean": -13.1367, + "std": 2.0311 + }, + "logp_gap": 2.4414, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -13.1367, + "logp_transplant_std": 2.0311 + }, + "1": { + "recovery_pct": 2.36, + "logp_transplant_mean": -13.0792, + "logp_transplant_std": 1.9806 + }, + "2": { + "recovery_pct": 3.16, + "logp_transplant_mean": -13.0596, + "logp_transplant_std": 2.0261 + }, + "3": { + "recovery_pct": 0.53, + "logp_transplant_mean": -13.1238, + "logp_transplant_std": 2.0499 + }, + "4": { + "recovery_pct": -0.11, + "logp_transplant_mean": -13.1394, + "logp_transplant_std": 2.0501 + }, + "5": { + "recovery_pct": 1.43, + "logp_transplant_mean": -13.1018, + "logp_transplant_std": 2.0504 + }, + "6": { + "recovery_pct": 5.33, + "logp_transplant_mean": -13.0067, + "logp_transplant_std": 2.0227 + }, + "7": { + "recovery_pct": 8.38, + "logp_transplant_mean": -12.9322, + "logp_transplant_std": 2.054 + }, + "8": { + "recovery_pct": 23.56, + "logp_transplant_mean": -12.5615, + "logp_transplant_std": 2.0684 + }, + "9": { + "recovery_pct": 27.23, + "logp_transplant_mean": -12.4719, + "logp_transplant_std": 2.1237 + }, + "10": { + "recovery_pct": 37.77, + "logp_transplant_mean": -12.2146, + "logp_transplant_std": 2.0986 + }, + "11": { + "recovery_pct": 39.8, + "logp_transplant_mean": -12.165, + "logp_transplant_std": 2.0208 + }, + "12": { + "recovery_pct": 62.72, + "logp_transplant_mean": -11.6054, + "logp_transplant_std": 2.2026 + }, + "13": { + "recovery_pct": 73.72, + "logp_transplant_mean": -11.3371, + "logp_transplant_std": 2.2133 + }, + "14": { + "recovery_pct": 74.94, + "logp_transplant_mean": -11.3071, + "logp_transplant_std": 2.2125 + }, + "15": { + "recovery_pct": 88.68, + "logp_transplant_mean": -10.9716, + "logp_transplant_std": 2.2882 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 16, + "d_model": 2048, + "model": "EleutherAI/pythia-1b", + "seed": 7, + "runtime_seconds": 163.0 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-2.8b_seed123.json b/data/e1_h3/EleutherAI--pythia-2.8b_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..e45167c7bd7bc88b2defd30c7e138c8e2425d110 --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-2.8b_seed123.json @@ -0,0 +1,191 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -14.896, + "std": 2.8198 + }, + "baseline_logp_d1000": { + "mean": -14.8718, + "std": 2.768 + }, + "logp_gap": -0.0243, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.8718, + "logp_transplant_std": 2.768 + }, + "1": { + "recovery_pct": 226.84, + "logp_transplant_mean": -14.9268, + "logp_transplant_std": 2.7485 + }, + "2": { + "recovery_pct": -685.9, + "logp_transplant_mean": -14.7054, + "logp_transplant_std": 2.6156 + }, + "3": { + "recovery_pct": 311.77, + "logp_transplant_mean": -14.9474, + "logp_transplant_std": 2.664 + }, + "4": { + "recovery_pct": -74.56, + "logp_transplant_mean": -14.8537, + "logp_transplant_std": 2.8103 + }, + "5": { + "recovery_pct": -80.88, + "logp_transplant_mean": -14.8522, + "logp_transplant_std": 2.764 + }, + "6": { + "recovery_pct": 640.24, + "logp_transplant_mean": -15.027, + "logp_transplant_std": 2.7019 + }, + "7": { + "recovery_pct": 510.06, + "logp_transplant_mean": -14.9955, + "logp_transplant_std": 2.777 + }, + "8": { + "recovery_pct": 261.18, + "logp_transplant_mean": -14.9351, + "logp_transplant_std": 2.7854 + }, + "9": { + "recovery_pct": 523.48, + "logp_transplant_mean": -14.9987, + "logp_transplant_std": 2.7159 + }, + "10": { + "recovery_pct": -351.06, + "logp_transplant_mean": -14.7866, + "logp_transplant_std": 2.6099 + }, + "11": { + "recovery_pct": 221.99, + "logp_transplant_mean": -14.9256, + "logp_transplant_std": 2.7896 + }, + "12": { + "recovery_pct": -253.17, + "logp_transplant_mean": -14.8104, + "logp_transplant_std": 2.5479 + }, + "13": { + "recovery_pct": -1547.85, + "logp_transplant_mean": -14.4964, + "logp_transplant_std": 2.9067 + }, + "14": { + "recovery_pct": -1643.17, + "logp_transplant_mean": -14.4733, + "logp_transplant_std": 2.4263 + }, + "15": { + "recovery_pct": -2296.15, + "logp_transplant_mean": -14.3149, + "logp_transplant_std": 2.7585 + }, + "16": { + "recovery_pct": -3058.3, + "logp_transplant_mean": -14.1301, + "logp_transplant_std": 2.4857 + }, + "17": { + "recovery_pct": -3119.84, + "logp_transplant_mean": -14.1152, + "logp_transplant_std": 2.4741 + }, + "18": { + "recovery_pct": -2303.57, + "logp_transplant_mean": -14.3131, + "logp_transplant_std": 2.722 + }, + "19": { + "recovery_pct": -4017.3, + "logp_transplant_mean": -13.8975, + "logp_transplant_std": 3.0785 + }, + "20": { + "recovery_pct": -3837.93, + "logp_transplant_mean": -13.941, + "logp_transplant_std": 3.2517 + }, + "21": { + "recovery_pct": -3766.23, + "logp_transplant_mean": -13.9584, + "logp_transplant_std": 2.9512 + }, + "22": { + "recovery_pct": -2979.19, + "logp_transplant_mean": -14.1493, + "logp_transplant_std": 2.8064 + }, + "23": { + "recovery_pct": -3342.41, + "logp_transplant_mean": -14.0612, + "logp_transplant_std": 3.0424 + }, + "24": { + "recovery_pct": -2151.72, + "logp_transplant_mean": -14.35, + "logp_transplant_std": 2.9267 + }, + "25": { + "recovery_pct": -2034.94, + "logp_transplant_mean": -14.3783, + "logp_transplant_std": 2.9472 + }, + "26": { + "recovery_pct": -1153.81, + "logp_transplant_mean": -14.592, + "logp_transplant_std": 2.9093 + }, + "27": { + "recovery_pct": -1178.87, + "logp_transplant_mean": -14.5859, + "logp_transplant_std": 2.9148 + }, + "28": { + "recovery_pct": -638.77, + "logp_transplant_mean": -14.7169, + "logp_transplant_std": 2.754 + }, + "29": { + "recovery_pct": -1135.57, + "logp_transplant_mean": -14.5964, + "logp_transplant_std": 2.8177 + }, + "30": { + "recovery_pct": -728.82, + "logp_transplant_mean": -14.695, + "logp_transplant_std": 2.7497 + }, + "31": { + "recovery_pct": -181.32, + "logp_transplant_mean": -14.8278, + "logp_transplant_std": 2.8554 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 32, + "d_model": 2560, + "model": "EleutherAI/pythia-2.8b", + "seed": 123, + "runtime_seconds": 1038.5 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-2.8b_seed42.json b/data/e1_h3/EleutherAI--pythia-2.8b_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..619c87caed7e50d14239878dc0c8997e81c6b01e --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-2.8b_seed42.json @@ -0,0 +1,191 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -14.8755, + "std": 2.7859 + }, + "baseline_logp_d1000": { + "mean": -14.8256, + "std": 2.527 + }, + "logp_gap": -0.05, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.8256, + "logp_transplant_std": 2.527 + }, + "1": { + "recovery_pct": 78.61, + "logp_transplant_mean": -14.8649, + "logp_transplant_std": 2.5669 + }, + "2": { + "recovery_pct": -373.24, + "logp_transplant_mean": -14.639, + "logp_transplant_std": 2.5252 + }, + "3": { + "recovery_pct": -304.78, + "logp_transplant_mean": -14.6733, + "logp_transplant_std": 2.4592 + }, + "4": { + "recovery_pct": 113.78, + "logp_transplant_mean": -14.8824, + "logp_transplant_std": 2.4183 + }, + "5": { + "recovery_pct": -269.64, + "logp_transplant_mean": -14.6908, + "logp_transplant_std": 2.3048 + }, + "6": { + "recovery_pct": -160.57, + "logp_transplant_mean": -14.7453, + "logp_transplant_std": 2.3683 + }, + "7": { + "recovery_pct": -470.9, + "logp_transplant_mean": -14.5902, + "logp_transplant_std": 2.2346 + }, + "8": { + "recovery_pct": -194.26, + "logp_transplant_mean": -14.7285, + "logp_transplant_std": 2.3823 + }, + "9": { + "recovery_pct": -379.04, + "logp_transplant_mean": -14.6361, + "logp_transplant_std": 2.1932 + }, + "10": { + "recovery_pct": -818.12, + "logp_transplant_mean": -14.4167, + "logp_transplant_std": 2.3253 + }, + "11": { + "recovery_pct": -785.01, + "logp_transplant_mean": -14.4333, + "logp_transplant_std": 1.9795 + }, + "12": { + "recovery_pct": -798.05, + "logp_transplant_mean": -14.4267, + "logp_transplant_std": 2.305 + }, + "13": { + "recovery_pct": -104.81, + "logp_transplant_mean": -14.7732, + "logp_transplant_std": 2.357 + }, + "14": { + "recovery_pct": 407.95, + "logp_transplant_mean": -15.0295, + "logp_transplant_std": 2.8019 + }, + "15": { + "recovery_pct": 71.89, + "logp_transplant_mean": -14.8615, + "logp_transplant_std": 2.6564 + }, + "16": { + "recovery_pct": -672.74, + "logp_transplant_mean": -14.4894, + "logp_transplant_std": 2.3578 + }, + "17": { + "recovery_pct": 112.29, + "logp_transplant_mean": -14.8817, + "logp_transplant_std": 2.3406 + }, + "18": { + "recovery_pct": -447.09, + "logp_transplant_mean": -14.6021, + "logp_transplant_std": 2.205 + }, + "19": { + "recovery_pct": -569.58, + "logp_transplant_mean": -14.5409, + "logp_transplant_std": 2.259 + }, + "20": { + "recovery_pct": -474.43, + "logp_transplant_mean": -14.5885, + "logp_transplant_std": 2.3099 + }, + "21": { + "recovery_pct": -739.27, + "logp_transplant_mean": -14.4561, + "logp_transplant_std": 2.477 + }, + "22": { + "recovery_pct": -1065.09, + "logp_transplant_mean": -14.2933, + "logp_transplant_std": 2.6142 + }, + "23": { + "recovery_pct": -600.65, + "logp_transplant_mean": -14.5254, + "logp_transplant_std": 2.5483 + }, + "24": { + "recovery_pct": -901.16, + "logp_transplant_mean": -14.3752, + "logp_transplant_std": 2.5421 + }, + "25": { + "recovery_pct": -536.13, + "logp_transplant_mean": -14.5576, + "logp_transplant_std": 2.4489 + }, + "26": { + "recovery_pct": -382.06, + "logp_transplant_mean": -14.6346, + "logp_transplant_std": 2.755 + }, + "27": { + "recovery_pct": -557.72, + "logp_transplant_mean": -14.5468, + "logp_transplant_std": 2.7981 + }, + "28": { + "recovery_pct": -625.42, + "logp_transplant_mean": -14.513, + "logp_transplant_std": 2.659 + }, + "29": { + "recovery_pct": -736.55, + "logp_transplant_mean": -14.4575, + "logp_transplant_std": 2.7427 + }, + "30": { + "recovery_pct": -567.96, + "logp_transplant_mean": -14.5417, + "logp_transplant_std": 2.7488 + }, + "31": { + "recovery_pct": -136.75, + "logp_transplant_mean": -14.7572, + "logp_transplant_std": 2.7043 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 32, + "d_model": 2560, + "model": "EleutherAI/pythia-2.8b", + "seed": 42, + "runtime_seconds": 1043.0 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-2.8b_seed7.json b/data/e1_h3/EleutherAI--pythia-2.8b_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..b27055a98cac43754477f33187e5e17c953f4a05 --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-2.8b_seed7.json @@ -0,0 +1,191 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -14.5505, + "std": 2.3669 + }, + "baseline_logp_d1000": { + "mean": -14.5263, + "std": 2.0644 + }, + "logp_gap": -0.0241, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.5263, + "logp_transplant_std": 2.0644 + }, + "1": { + "recovery_pct": 424.05, + "logp_transplant_mean": -14.6286, + "logp_transplant_std": 2.0015 + }, + "2": { + "recovery_pct": 296.44, + "logp_transplant_mean": -14.5978, + "logp_transplant_std": 2.0379 + }, + "3": { + "recovery_pct": 303.01, + "logp_transplant_mean": -14.5994, + "logp_transplant_std": 2.0261 + }, + "4": { + "recovery_pct": 103.81, + "logp_transplant_mean": -14.5514, + "logp_transplant_std": 2.0235 + }, + "5": { + "recovery_pct": 164.62, + "logp_transplant_mean": -14.566, + "logp_transplant_std": 2.0006 + }, + "6": { + "recovery_pct": 181.55, + "logp_transplant_mean": -14.5701, + "logp_transplant_std": 1.9312 + }, + "7": { + "recovery_pct": -249.17, + "logp_transplant_mean": -14.4663, + "logp_transplant_std": 1.9624 + }, + "8": { + "recovery_pct": 633.83, + "logp_transplant_mean": -14.6791, + "logp_transplant_std": 2.1509 + }, + "9": { + "recovery_pct": -724.02, + "logp_transplant_mean": -14.3518, + "logp_transplant_std": 1.7872 + }, + "10": { + "recovery_pct": -221.86, + "logp_transplant_mean": -14.4729, + "logp_transplant_std": 1.9578 + }, + "11": { + "recovery_pct": 193.14, + "logp_transplant_mean": -14.5729, + "logp_transplant_std": 2.2061 + }, + "12": { + "recovery_pct": -939.91, + "logp_transplant_mean": -14.2998, + "logp_transplant_std": 2.0577 + }, + "13": { + "recovery_pct": -1738.0, + "logp_transplant_mean": -14.1074, + "logp_transplant_std": 1.9392 + }, + "14": { + "recovery_pct": 226.99, + "logp_transplant_mean": -14.5811, + "logp_transplant_std": 2.1929 + }, + "15": { + "recovery_pct": 232.18, + "logp_transplant_mean": -14.5823, + "logp_transplant_std": 2.4139 + }, + "16": { + "recovery_pct": -669.25, + "logp_transplant_mean": -14.365, + "logp_transplant_std": 2.2685 + }, + "17": { + "recovery_pct": -1430.71, + "logp_transplant_mean": -14.1814, + "logp_transplant_std": 2.0136 + }, + "18": { + "recovery_pct": -2285.71, + "logp_transplant_mean": -13.9753, + "logp_transplant_std": 2.0579 + }, + "19": { + "recovery_pct": -2216.02, + "logp_transplant_mean": -13.9921, + "logp_transplant_std": 2.3351 + }, + "20": { + "recovery_pct": -1722.71, + "logp_transplant_mean": -14.111, + "logp_transplant_std": 2.395 + }, + "21": { + "recovery_pct": -714.96, + "logp_transplant_mean": -14.354, + "logp_transplant_std": 2.7969 + }, + "22": { + "recovery_pct": -2319.52, + "logp_transplant_mean": -13.9672, + "logp_transplant_std": 2.3644 + }, + "23": { + "recovery_pct": -3224.96, + "logp_transplant_mean": -13.7489, + "logp_transplant_std": 2.2707 + }, + "24": { + "recovery_pct": -2246.88, + "logp_transplant_mean": -13.9847, + "logp_transplant_std": 2.474 + }, + "25": { + "recovery_pct": -2423.99, + "logp_transplant_mean": -13.942, + "logp_transplant_std": 2.4583 + }, + "26": { + "recovery_pct": -1875.13, + "logp_transplant_mean": -14.0743, + "logp_transplant_std": 2.221 + }, + "27": { + "recovery_pct": -1567.33, + "logp_transplant_mean": -14.1485, + "logp_transplant_std": 2.1927 + }, + "28": { + "recovery_pct": -1738.19, + "logp_transplant_mean": -14.1073, + "logp_transplant_std": 2.4385 + }, + "29": { + "recovery_pct": -1932.71, + "logp_transplant_mean": -14.0604, + "logp_transplant_std": 2.5116 + }, + "30": { + "recovery_pct": -1596.15, + "logp_transplant_mean": -14.1415, + "logp_transplant_std": 2.4858 + }, + "31": { + "recovery_pct": -787.0, + "logp_transplant_mean": -14.3366, + "logp_transplant_std": 2.3893 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 32, + "d_model": 2560, + "model": "EleutherAI/pythia-2.8b", + "seed": 7, + "runtime_seconds": 1034.4 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-410m_seed123.json b/data/e1_h3/EleutherAI--pythia-410m_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..3182bbc03dd64224d1317351d2f2276c967dfeeb --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-410m_seed123.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 100, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -15.3925, + "std": 3.2501 + }, + "baseline_logp_d1000": { + "mean": -14.657, + "std": 2.4179 + }, + "logp_gap": -0.7355, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.657, + "logp_transplant_std": 2.4179 + }, + "1": { + "recovery_pct": -29.31, + "logp_transplant_mean": -14.4414, + "logp_transplant_std": 2.5807 + }, + "2": { + "recovery_pct": -16.5, + "logp_transplant_mean": -14.5356, + "logp_transplant_std": 2.572 + }, + "3": { + "recovery_pct": 63.8, + "logp_transplant_mean": -15.1262, + "logp_transplant_std": 2.5965 + }, + "4": { + "recovery_pct": 67.29, + "logp_transplant_mean": -15.1519, + "logp_transplant_std": 2.6477 + }, + "5": { + "recovery_pct": 75.26, + "logp_transplant_mean": -15.2105, + "logp_transplant_std": 2.6668 + }, + "6": { + "recovery_pct": 48.57, + "logp_transplant_mean": -15.0142, + "logp_transplant_std": 2.593 + }, + "7": { + "recovery_pct": 63.91, + "logp_transplant_mean": -15.1271, + "logp_transplant_std": 2.6332 + }, + "8": { + "recovery_pct": 50.95, + "logp_transplant_mean": -15.0317, + "logp_transplant_std": 2.6447 + }, + "9": { + "recovery_pct": 10.37, + "logp_transplant_mean": -14.7332, + "logp_transplant_std": 2.7151 + }, + "10": { + "recovery_pct": 50.65, + "logp_transplant_mean": -15.0295, + "logp_transplant_std": 2.6952 + }, + "11": { + "recovery_pct": 43.0, + "logp_transplant_mean": -14.9732, + "logp_transplant_std": 2.9789 + }, + "12": { + "recovery_pct": 75.95, + "logp_transplant_mean": -15.2156, + "logp_transplant_std": 2.8685 + }, + "13": { + "recovery_pct": 80.44, + "logp_transplant_mean": -15.2486, + "logp_transplant_std": 2.8648 + }, + "14": { + "recovery_pct": 84.22, + "logp_transplant_mean": -15.2764, + "logp_transplant_std": 2.6435 + }, + "15": { + "recovery_pct": 71.74, + "logp_transplant_mean": -15.1847, + "logp_transplant_std": 2.4936 + }, + "16": { + "recovery_pct": 74.92, + "logp_transplant_mean": -15.208, + "logp_transplant_std": 2.7362 + }, + "17": { + "recovery_pct": 108.35, + "logp_transplant_mean": -15.4539, + "logp_transplant_std": 3.022 + }, + "18": { + "recovery_pct": 109.66, + "logp_transplant_mean": -15.4636, + "logp_transplant_std": 3.1044 + }, + "19": { + "recovery_pct": 55.39, + "logp_transplant_mean": -15.0644, + "logp_transplant_std": 3.1129 + }, + "20": { + "recovery_pct": 88.71, + "logp_transplant_mean": -15.3094, + "logp_transplant_std": 3.2357 + }, + "21": { + "recovery_pct": 93.21, + "logp_transplant_mean": -15.3425, + "logp_transplant_std": 3.1739 + }, + "22": { + "recovery_pct": 89.72, + "logp_transplant_mean": -15.3169, + "logp_transplant_std": 3.3374 + }, + "23": { + "recovery_pct": 102.36, + "logp_transplant_mean": -15.4099, + "logp_transplant_std": 3.401 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 24, + "d_model": 1024, + "model": "EleutherAI/pythia-410m", + "seed": 123, + "runtime_seconds": 330.9 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-410m_seed42.json b/data/e1_h3/EleutherAI--pythia-410m_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..29b78221455b965392c88101b1fc194d3e7a3244 --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-410m_seed42.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 100, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -14.6451, + "std": 3.0988 + }, + "baseline_logp_d1000": { + "mean": -14.6291, + "std": 2.388 + }, + "logp_gap": -0.0161, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.6291, + "logp_transplant_std": 2.388 + }, + "1": { + "recovery_pct": -5050.33, + "logp_transplant_mean": -13.8177, + "logp_transplant_std": 2.233 + }, + "2": { + "recovery_pct": -3127.33, + "logp_transplant_mean": -14.1266, + "logp_transplant_std": 2.35 + }, + "3": { + "recovery_pct": 1832.75, + "logp_transplant_mean": -14.9235, + "logp_transplant_std": 2.2994 + }, + "4": { + "recovery_pct": 945.74, + "logp_transplant_mean": -14.781, + "logp_transplant_std": 2.3575 + }, + "5": { + "recovery_pct": 1694.84, + "logp_transplant_mean": -14.9014, + "logp_transplant_std": 2.4098 + }, + "6": { + "recovery_pct": 2094.07, + "logp_transplant_mean": -14.9655, + "logp_transplant_std": 2.3604 + }, + "7": { + "recovery_pct": 1837.22, + "logp_transplant_mean": -14.9242, + "logp_transplant_std": 2.4151 + }, + "8": { + "recovery_pct": -162.02, + "logp_transplant_mean": -14.603, + "logp_transplant_std": 2.5811 + }, + "9": { + "recovery_pct": 524.54, + "logp_transplant_mean": -14.7133, + "logp_transplant_std": 2.5711 + }, + "10": { + "recovery_pct": -1133.04, + "logp_transplant_mean": -14.447, + "logp_transplant_std": 2.5077 + }, + "11": { + "recovery_pct": 294.75, + "logp_transplant_mean": -14.6764, + "logp_transplant_std": 2.5075 + }, + "12": { + "recovery_pct": 63.72, + "logp_transplant_mean": -14.6393, + "logp_transplant_std": 2.5217 + }, + "13": { + "recovery_pct": 279.82, + "logp_transplant_mean": -14.674, + "logp_transplant_std": 2.3902 + }, + "14": { + "recovery_pct": 90.83, + "logp_transplant_mean": -14.6437, + "logp_transplant_std": 2.4973 + }, + "15": { + "recovery_pct": 356.09, + "logp_transplant_mean": -14.6863, + "logp_transplant_std": 2.6265 + }, + "16": { + "recovery_pct": 2421.84, + "logp_transplant_mean": -15.0182, + "logp_transplant_std": 2.9079 + }, + "17": { + "recovery_pct": 2248.68, + "logp_transplant_mean": -14.9903, + "logp_transplant_std": 3.0452 + }, + "18": { + "recovery_pct": 1279.96, + "logp_transplant_mean": -14.8347, + "logp_transplant_std": 2.8254 + }, + "19": { + "recovery_pct": -1248.61, + "logp_transplant_mean": -14.4285, + "logp_transplant_std": 2.8562 + }, + "20": { + "recovery_pct": -7.37, + "logp_transplant_mean": -14.6279, + "logp_transplant_std": 2.8882 + }, + "21": { + "recovery_pct": 401.03, + "logp_transplant_mean": -14.6935, + "logp_transplant_std": 2.9714 + }, + "22": { + "recovery_pct": 558.51, + "logp_transplant_mean": -14.7188, + "logp_transplant_std": 3.2055 + }, + "23": { + "recovery_pct": 116.62, + "logp_transplant_mean": -14.6478, + "logp_transplant_std": 3.0985 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 24, + "d_model": 1024, + "model": "EleutherAI/pythia-410m", + "seed": 42, + "runtime_seconds": 331.3 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-410m_seed7.json b/data/e1_h3/EleutherAI--pythia-410m_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..8ebc0b4617f1aaab9b576acab7e1446b0a59842b --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-410m_seed7.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 100, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -15.3805, + "std": 3.0915 + }, + "baseline_logp_d1000": { + "mean": -14.9379, + "std": 2.6239 + }, + "logp_gap": -0.4425, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.9379, + "logp_transplant_std": 2.6239 + }, + "1": { + "recovery_pct": -67.39, + "logp_transplant_mean": -14.6397, + "logp_transplant_std": 2.6372 + }, + "2": { + "recovery_pct": -66.87, + "logp_transplant_mean": -14.642, + "logp_transplant_std": 2.4338 + }, + "3": { + "recovery_pct": 96.35, + "logp_transplant_mean": -15.3643, + "logp_transplant_std": 2.5628 + }, + "4": { + "recovery_pct": 88.33, + "logp_transplant_mean": -15.3288, + "logp_transplant_std": 2.5043 + }, + "5": { + "recovery_pct": 77.98, + "logp_transplant_mean": -15.283, + "logp_transplant_std": 2.4987 + }, + "6": { + "recovery_pct": 101.2, + "logp_transplant_mean": -15.3858, + "logp_transplant_std": 2.5212 + }, + "7": { + "recovery_pct": 73.33, + "logp_transplant_mean": -15.2624, + "logp_transplant_std": 2.5733 + }, + "8": { + "recovery_pct": 77.37, + "logp_transplant_mean": -15.2803, + "logp_transplant_std": 2.5504 + }, + "9": { + "recovery_pct": 63.47, + "logp_transplant_mean": -15.2188, + "logp_transplant_std": 2.4771 + }, + "10": { + "recovery_pct": 58.39, + "logp_transplant_mean": -15.1963, + "logp_transplant_std": 2.4966 + }, + "11": { + "recovery_pct": 70.46, + "logp_transplant_mean": -15.2497, + "logp_transplant_std": 2.3882 + }, + "12": { + "recovery_pct": 128.46, + "logp_transplant_mean": -15.5064, + "logp_transplant_std": 2.407 + }, + "13": { + "recovery_pct": 88.46, + "logp_transplant_mean": -15.3294, + "logp_transplant_std": 2.4682 + }, + "14": { + "recovery_pct": 99.88, + "logp_transplant_mean": -15.3799, + "logp_transplant_std": 2.5274 + }, + "15": { + "recovery_pct": 148.98, + "logp_transplant_mean": -15.5972, + "logp_transplant_std": 2.4799 + }, + "16": { + "recovery_pct": 199.79, + "logp_transplant_mean": -15.822, + "logp_transplant_std": 2.7001 + }, + "17": { + "recovery_pct": 187.15, + "logp_transplant_mean": -15.7661, + "logp_transplant_std": 2.8041 + }, + "18": { + "recovery_pct": 140.55, + "logp_transplant_mean": -15.5599, + "logp_transplant_std": 2.9837 + }, + "19": { + "recovery_pct": 107.73, + "logp_transplant_mean": -15.4146, + "logp_transplant_std": 2.9998 + }, + "20": { + "recovery_pct": 108.3, + "logp_transplant_mean": -15.4172, + "logp_transplant_std": 3.0468 + }, + "21": { + "recovery_pct": 86.7, + "logp_transplant_mean": -15.3216, + "logp_transplant_std": 3.1507 + }, + "22": { + "recovery_pct": 113.15, + "logp_transplant_mean": -15.4386, + "logp_transplant_std": 3.1387 + }, + "23": { + "recovery_pct": 85.39, + "logp_transplant_mean": -15.3158, + "logp_transplant_std": 3.1329 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 24, + "d_model": 1024, + "model": "EleutherAI/pythia-410m", + "seed": 7, + "runtime_seconds": 331.8 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-70m_seed123.json b/data/e1_h3/EleutherAI--pythia-70m_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..1e1a45c2da1381b8a807505e450ee8e4ac2282fc --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-70m_seed123.json @@ -0,0 +1,61 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -13.0993, + "std": 3.4102 + }, + "baseline_logp_d1000": { + "mean": -15.0573, + "std": 3.4794 + }, + "logp_gap": 1.958, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -15.0573, + "logp_transplant_std": 3.4794 + }, + "1": { + "recovery_pct": 32.43, + "logp_transplant_mean": -14.4223, + "logp_transplant_std": 2.6432 + }, + "2": { + "recovery_pct": 29.62, + "logp_transplant_mean": -14.4773, + "logp_transplant_std": 2.7298 + }, + "3": { + "recovery_pct": 33.57, + "logp_transplant_mean": -14.4001, + "logp_transplant_std": 2.4779 + }, + "4": { + "recovery_pct": 109.94, + "logp_transplant_mean": -12.9046, + "logp_transplant_std": 3.3631 + }, + "5": { + "recovery_pct": 108.91, + "logp_transplant_mean": -12.9248, + "logp_transplant_std": 3.3961 + } + }, + "L_crit_90": 4, + "L_crit_99": 4, + "alpha_90": 0.6667, + "alpha_99": 0.6667, + "recovery_at_Lcrit": 1.0994, + "n_layers": 6, + "d_model": 512, + "model": "EleutherAI/pythia-70m", + "seed": 123, + "runtime_seconds": 6.7 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-70m_seed42.json b/data/e1_h3/EleutherAI--pythia-70m_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..4aa9fc6ca1868a380b4dbd6942b62d79caa2779d --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-70m_seed42.json @@ -0,0 +1,61 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -15.7736, + "std": 14.0313 + }, + "baseline_logp_d1000": { + "mean": -16.482, + "std": 11.7338 + }, + "logp_gap": 0.7085, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -16.482, + "logp_transplant_std": 11.7338 + }, + "1": { + "recovery_pct": -167.46, + "logp_transplant_mean": -17.6685, + "logp_transplant_std": 16.7109 + }, + "2": { + "recovery_pct": -70.17, + "logp_transplant_mean": -16.9792, + "logp_transplant_std": 13.0732 + }, + "3": { + "recovery_pct": -63.83, + "logp_transplant_mean": -16.9343, + "logp_transplant_std": 12.0846 + }, + "4": { + "recovery_pct": 129.49, + "logp_transplant_mean": -15.5646, + "logp_transplant_std": 13.3919 + }, + "5": { + "recovery_pct": 95.89, + "logp_transplant_mean": -15.8027, + "logp_transplant_std": 14.5107 + } + }, + "L_crit_90": 4, + "L_crit_99": 4, + "alpha_90": 0.6667, + "alpha_99": 0.6667, + "recovery_at_Lcrit": 1.2949, + "n_layers": 6, + "d_model": 512, + "model": "EleutherAI/pythia-70m", + "seed": 42, + "runtime_seconds": 6.5 +} \ No newline at end of file diff --git a/data/e1_h3/EleutherAI--pythia-70m_seed7.json b/data/e1_h3/EleutherAI--pythia-70m_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..e52042869cedf961237096694a8f0d087f70fe46 --- /dev/null +++ b/data/e1_h3/EleutherAI--pythia-70m_seed7.json @@ -0,0 +1,61 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -31.5123, + "std": 114.2086 + }, + "baseline_logp_d1000": { + "mean": -31.882, + "std": 115.6761 + }, + "logp_gap": 0.3697, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -31.882, + "logp_transplant_std": 115.6761 + }, + "1": { + "recovery_pct": -129.99, + "logp_transplant_mean": -32.3625, + "logp_transplant_std": 114.8515 + }, + "2": { + "recovery_pct": -16.62, + "logp_transplant_mean": -31.9434, + "logp_transplant_std": 112.2661 + }, + "3": { + "recovery_pct": -73.21, + "logp_transplant_mean": -32.1526, + "logp_transplant_std": 114.1947 + }, + "4": { + "recovery_pct": 170.43, + "logp_transplant_mean": -31.2519, + "logp_transplant_std": 115.2078 + }, + "5": { + "recovery_pct": 140.77, + "logp_transplant_mean": -31.3616, + "logp_transplant_std": 114.4732 + } + }, + "L_crit_90": 4, + "L_crit_99": 4, + "alpha_90": 0.6667, + "alpha_99": 0.6667, + "recovery_at_Lcrit": 1.7043, + "n_layers": 6, + "d_model": 512, + "model": "EleutherAI/pythia-70m", + "seed": 7, + "runtime_seconds": 6.3 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2-large_seed123.json b/data/e1_h3/gpt2-large_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..f7b0e29035fa769a62506376dcd66ddd2fad0ab4 --- /dev/null +++ b/data/e1_h3/gpt2-large_seed123.json @@ -0,0 +1,211 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -5.89, + "std": 2.385 + }, + "baseline_logp_d1000": { + "mean": -10.222, + "std": 1.5212 + }, + "logp_gap": 4.332, + "layer_sweep": { + "0": { + "recovery_pct": 3.78, + "logp_transplant_mean": -10.0582, + "logp_transplant_std": 1.5721 + }, + "1": { + "recovery_pct": -2.73, + "logp_transplant_mean": -10.3403, + "logp_transplant_std": 1.3639 + }, + "2": { + "recovery_pct": -2.43, + "logp_transplant_mean": -10.3273, + "logp_transplant_std": 1.6248 + }, + "3": { + "recovery_pct": 0.38, + "logp_transplant_mean": -10.2057, + "logp_transplant_std": 1.6892 + }, + "4": { + "recovery_pct": 3.83, + "logp_transplant_mean": -10.0562, + "logp_transplant_std": 1.7289 + }, + "5": { + "recovery_pct": 8.03, + "logp_transplant_mean": -9.8741, + "logp_transplant_std": 1.7546 + }, + "6": { + "recovery_pct": 4.43, + "logp_transplant_mean": -10.0301, + "logp_transplant_std": 1.6993 + }, + "7": { + "recovery_pct": 2.43, + "logp_transplant_mean": -10.1167, + "logp_transplant_std": 1.6938 + }, + "8": { + "recovery_pct": -0.13, + "logp_transplant_mean": -10.2275, + "logp_transplant_std": 1.6459 + }, + "9": { + "recovery_pct": -5.89, + "logp_transplant_mean": -10.4772, + "logp_transplant_std": 1.5576 + }, + "10": { + "recovery_pct": -3.67, + "logp_transplant_mean": -10.3808, + "logp_transplant_std": 1.5886 + }, + "11": { + "recovery_pct": -4.13, + "logp_transplant_mean": -10.4007, + "logp_transplant_std": 1.5926 + }, + "12": { + "recovery_pct": 1.27, + "logp_transplant_mean": -10.167, + "logp_transplant_std": 1.6214 + }, + "13": { + "recovery_pct": 2.98, + "logp_transplant_mean": -10.0928, + "logp_transplant_std": 1.6084 + }, + "14": { + "recovery_pct": 5.84, + "logp_transplant_mean": -9.9692, + "logp_transplant_std": 1.6272 + }, + "15": { + "recovery_pct": 2.38, + "logp_transplant_mean": -10.1187, + "logp_transplant_std": 1.5275 + }, + "16": { + "recovery_pct": 0.74, + "logp_transplant_mean": -10.1898, + "logp_transplant_std": 1.5505 + }, + "17": { + "recovery_pct": 0.38, + "logp_transplant_mean": -10.2057, + "logp_transplant_std": 1.553 + }, + "18": { + "recovery_pct": 12.75, + "logp_transplant_mean": -9.6697, + "logp_transplant_std": 1.6674 + }, + "19": { + "recovery_pct": 23.41, + "logp_transplant_mean": -9.2076, + "logp_transplant_std": 1.8 + }, + "20": { + "recovery_pct": 27.12, + "logp_transplant_mean": -9.0471, + "logp_transplant_std": 1.7964 + }, + "21": { + "recovery_pct": 19.51, + "logp_transplant_mean": -9.377, + "logp_transplant_std": 1.6327 + }, + "22": { + "recovery_pct": 25.56, + "logp_transplant_mean": -9.1147, + "logp_transplant_std": 1.7371 + }, + "23": { + "recovery_pct": 32.73, + "logp_transplant_mean": -8.804, + "logp_transplant_std": 1.7839 + }, + "24": { + "recovery_pct": 36.59, + "logp_transplant_mean": -8.637, + "logp_transplant_std": 1.8559 + }, + "25": { + "recovery_pct": 43.98, + "logp_transplant_mean": -8.3166, + "logp_transplant_std": 2.066 + }, + "26": { + "recovery_pct": 47.69, + "logp_transplant_mean": -8.1562, + "logp_transplant_std": 2.1376 + }, + "27": { + "recovery_pct": 55.73, + "logp_transplant_mean": -7.8078, + "logp_transplant_std": 2.2994 + }, + "28": { + "recovery_pct": 60.59, + "logp_transplant_mean": -7.5972, + "logp_transplant_std": 2.2917 + }, + "29": { + "recovery_pct": 66.22, + "logp_transplant_mean": -7.3534, + "logp_transplant_std": 2.303 + }, + "30": { + "recovery_pct": 72.07, + "logp_transplant_mean": -7.0999, + "logp_transplant_std": 2.3257 + }, + "31": { + "recovery_pct": 77.97, + "logp_transplant_mean": -6.8442, + "logp_transplant_std": 2.3139 + }, + "32": { + "recovery_pct": 84.02, + "logp_transplant_mean": -6.5822, + "logp_transplant_std": 2.272 + }, + "33": { + "recovery_pct": 92.62, + "logp_transplant_mean": -6.2099, + "logp_transplant_std": 2.3261 + }, + "34": { + "recovery_pct": 97.53, + "logp_transplant_mean": -5.9972, + "logp_transplant_std": 2.3601 + }, + "35": { + "recovery_pct": 100.43, + "logp_transplant_mean": -5.8713, + "logp_transplant_std": 2.4116 + } + }, + "L_crit_90": 33, + "L_crit_99": 35, + "alpha_90": 0.9167, + "alpha_99": 0.9722, + "recovery_at_Lcrit": 0.9262, + "n_layers": 36, + "d_model": 1280, + "model": "gpt2-large", + "seed": 123, + "runtime_seconds": 511.3 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2-large_seed42.json b/data/e1_h3/gpt2-large_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..1c9f2260d1fdabfe54398ec20e182ba72adf6439 --- /dev/null +++ b/data/e1_h3/gpt2-large_seed42.json @@ -0,0 +1,211 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -5.973, + "std": 2.7914 + }, + "baseline_logp_d1000": { + "mean": -10.0461, + "std": 1.8416 + }, + "logp_gap": 4.0732, + "layer_sweep": { + "0": { + "recovery_pct": 0.73, + "logp_transplant_mean": -10.0165, + "logp_transplant_std": 1.869 + }, + "1": { + "recovery_pct": -3.21, + "logp_transplant_mean": -10.177, + "logp_transplant_std": 1.6988 + }, + "2": { + "recovery_pct": -4.58, + "logp_transplant_mean": -10.2329, + "logp_transplant_std": 1.657 + }, + "3": { + "recovery_pct": -5.58, + "logp_transplant_mean": -10.2736, + "logp_transplant_std": 1.5692 + }, + "4": { + "recovery_pct": 4.05, + "logp_transplant_mean": -9.8813, + "logp_transplant_std": 1.9463 + }, + "5": { + "recovery_pct": 6.95, + "logp_transplant_mean": -9.763, + "logp_transplant_std": 1.9088 + }, + "6": { + "recovery_pct": 1.72, + "logp_transplant_mean": -9.9763, + "logp_transplant_std": 1.7947 + }, + "7": { + "recovery_pct": -1.27, + "logp_transplant_mean": -10.0977, + "logp_transplant_std": 1.762 + }, + "8": { + "recovery_pct": -3.37, + "logp_transplant_mean": -10.1833, + "logp_transplant_std": 1.7802 + }, + "9": { + "recovery_pct": -7.98, + "logp_transplant_mean": -10.3711, + "logp_transplant_std": 1.7532 + }, + "10": { + "recovery_pct": -5.9, + "logp_transplant_mean": -10.2864, + "logp_transplant_std": 1.8161 + }, + "11": { + "recovery_pct": -5.35, + "logp_transplant_mean": -10.2641, + "logp_transplant_std": 1.8444 + }, + "12": { + "recovery_pct": -0.13, + "logp_transplant_mean": -10.0516, + "logp_transplant_std": 1.8866 + }, + "13": { + "recovery_pct": 0.98, + "logp_transplant_mean": -10.006, + "logp_transplant_std": 1.8753 + }, + "14": { + "recovery_pct": 3.22, + "logp_transplant_mean": -9.9151, + "logp_transplant_std": 1.8654 + }, + "15": { + "recovery_pct": -0.14, + "logp_transplant_mean": -10.0517, + "logp_transplant_std": 1.834 + }, + "16": { + "recovery_pct": -0.64, + "logp_transplant_mean": -10.0724, + "logp_transplant_std": 1.8035 + }, + "17": { + "recovery_pct": 0.13, + "logp_transplant_mean": -10.041, + "logp_transplant_std": 1.903 + }, + "18": { + "recovery_pct": 10.95, + "logp_transplant_mean": -9.6003, + "logp_transplant_std": 2.0453 + }, + "19": { + "recovery_pct": 22.56, + "logp_transplant_mean": -9.1271, + "logp_transplant_std": 2.0656 + }, + "20": { + "recovery_pct": 26.65, + "logp_transplant_mean": -8.9608, + "logp_transplant_std": 2.1668 + }, + "21": { + "recovery_pct": 24.56, + "logp_transplant_mean": -9.0457, + "logp_transplant_std": 1.9832 + }, + "22": { + "recovery_pct": 29.34, + "logp_transplant_mean": -8.8511, + "logp_transplant_std": 2.1047 + }, + "23": { + "recovery_pct": 36.9, + "logp_transplant_mean": -8.543, + "logp_transplant_std": 2.1973 + }, + "24": { + "recovery_pct": 39.33, + "logp_transplant_mean": -8.4444, + "logp_transplant_std": 2.2375 + }, + "25": { + "recovery_pct": 45.19, + "logp_transplant_mean": -8.2053, + "logp_transplant_std": 2.4138 + }, + "26": { + "recovery_pct": 45.12, + "logp_transplant_mean": -8.2084, + "logp_transplant_std": 2.4853 + }, + "27": { + "recovery_pct": 52.73, + "logp_transplant_mean": -7.8985, + "logp_transplant_std": 2.6234 + }, + "28": { + "recovery_pct": 59.13, + "logp_transplant_mean": -7.6375, + "logp_transplant_std": 2.623 + }, + "29": { + "recovery_pct": 64.24, + "logp_transplant_mean": -7.4297, + "logp_transplant_std": 2.6947 + }, + "30": { + "recovery_pct": 70.66, + "logp_transplant_mean": -7.1681, + "logp_transplant_std": 2.7442 + }, + "31": { + "recovery_pct": 78.28, + "logp_transplant_mean": -6.8578, + "logp_transplant_std": 2.8388 + }, + "32": { + "recovery_pct": 84.29, + "logp_transplant_mean": -6.613, + "logp_transplant_std": 2.8354 + }, + "33": { + "recovery_pct": 91.44, + "logp_transplant_mean": -6.3216, + "logp_transplant_std": 2.83 + }, + "34": { + "recovery_pct": 97.63, + "logp_transplant_mean": -6.0695, + "logp_transplant_std": 2.8588 + }, + "35": { + "recovery_pct": 100.04, + "logp_transplant_mean": -5.9713, + "logp_transplant_std": 2.8352 + } + }, + "L_crit_90": 33, + "L_crit_99": 35, + "alpha_90": 0.9167, + "alpha_99": 0.9722, + "recovery_at_Lcrit": 0.9144, + "n_layers": 36, + "d_model": 1280, + "model": "gpt2-large", + "seed": 42, + "runtime_seconds": 511.0 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2-large_seed7.json b/data/e1_h3/gpt2-large_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..199c198baa0379d5e481062910afdaee631a3a19 --- /dev/null +++ b/data/e1_h3/gpt2-large_seed7.json @@ -0,0 +1,211 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -6.0858, + "std": 2.6161 + }, + "baseline_logp_d1000": { + "mean": -10.0017, + "std": 1.8738 + }, + "logp_gap": 3.9158, + "layer_sweep": { + "0": { + "recovery_pct": 2.19, + "logp_transplant_mean": -9.916, + "logp_transplant_std": 1.8715 + }, + "1": { + "recovery_pct": -2.05, + "logp_transplant_mean": -10.082, + "logp_transplant_std": 1.8003 + }, + "2": { + "recovery_pct": -5.31, + "logp_transplant_mean": -10.2098, + "logp_transplant_std": 1.7483 + }, + "3": { + "recovery_pct": -4.48, + "logp_transplant_mean": -10.177, + "logp_transplant_std": 1.7624 + }, + "4": { + "recovery_pct": -1.49, + "logp_transplant_mean": -10.0599, + "logp_transplant_std": 1.8789 + }, + "5": { + "recovery_pct": 6.39, + "logp_transplant_mean": -9.7515, + "logp_transplant_std": 2.0801 + }, + "6": { + "recovery_pct": 1.93, + "logp_transplant_mean": -9.9263, + "logp_transplant_std": 2.0373 + }, + "7": { + "recovery_pct": 0.93, + "logp_transplant_mean": -9.9651, + "logp_transplant_std": 2.0156 + }, + "8": { + "recovery_pct": -1.5, + "logp_transplant_mean": -10.0602, + "logp_transplant_std": 1.9984 + }, + "9": { + "recovery_pct": -6.05, + "logp_transplant_mean": -10.2386, + "logp_transplant_std": 1.9155 + }, + "10": { + "recovery_pct": -3.56, + "logp_transplant_mean": -10.1411, + "logp_transplant_std": 1.9651 + }, + "11": { + "recovery_pct": -3.66, + "logp_transplant_mean": -10.1448, + "logp_transplant_std": 1.9578 + }, + "12": { + "recovery_pct": 0.72, + "logp_transplant_mean": -9.9733, + "logp_transplant_std": 2.0 + }, + "13": { + "recovery_pct": 1.84, + "logp_transplant_mean": -9.9295, + "logp_transplant_std": 1.9409 + }, + "14": { + "recovery_pct": 3.93, + "logp_transplant_mean": -9.8479, + "logp_transplant_std": 1.9835 + }, + "15": { + "recovery_pct": 1.78, + "logp_transplant_mean": -9.932, + "logp_transplant_std": 1.9319 + }, + "16": { + "recovery_pct": -0.16, + "logp_transplant_mean": -10.0078, + "logp_transplant_std": 1.9409 + }, + "17": { + "recovery_pct": -1.22, + "logp_transplant_mean": -10.0494, + "logp_transplant_std": 1.953 + }, + "18": { + "recovery_pct": 11.49, + "logp_transplant_mean": -9.5517, + "logp_transplant_std": 2.0252 + }, + "19": { + "recovery_pct": 23.19, + "logp_transplant_mean": -9.0935, + "logp_transplant_std": 2.0581 + }, + "20": { + "recovery_pct": 25.99, + "logp_transplant_mean": -8.9838, + "logp_transplant_std": 2.0151 + }, + "21": { + "recovery_pct": 20.92, + "logp_transplant_mean": -9.1823, + "logp_transplant_std": 1.8835 + }, + "22": { + "recovery_pct": 24.3, + "logp_transplant_mean": -9.0503, + "logp_transplant_std": 2.0546 + }, + "23": { + "recovery_pct": 32.85, + "logp_transplant_mean": -8.7154, + "logp_transplant_std": 2.1969 + }, + "24": { + "recovery_pct": 35.36, + "logp_transplant_mean": -8.6169, + "logp_transplant_std": 2.249 + }, + "25": { + "recovery_pct": 43.05, + "logp_transplant_mean": -8.3159, + "logp_transplant_std": 2.299 + }, + "26": { + "recovery_pct": 45.27, + "logp_transplant_mean": -8.2288, + "logp_transplant_std": 2.2486 + }, + "27": { + "recovery_pct": 53.79, + "logp_transplant_mean": -7.8955, + "logp_transplant_std": 2.3145 + }, + "28": { + "recovery_pct": 59.53, + "logp_transplant_mean": -7.6707, + "logp_transplant_std": 2.3374 + }, + "29": { + "recovery_pct": 65.41, + "logp_transplant_mean": -7.4402, + "logp_transplant_std": 2.313 + }, + "30": { + "recovery_pct": 71.15, + "logp_transplant_mean": -7.2154, + "logp_transplant_std": 2.4051 + }, + "31": { + "recovery_pct": 80.39, + "logp_transplant_mean": -6.8537, + "logp_transplant_std": 2.4885 + }, + "32": { + "recovery_pct": 85.98, + "logp_transplant_mean": -6.6348, + "logp_transplant_std": 2.5288 + }, + "33": { + "recovery_pct": 93.68, + "logp_transplant_mean": -6.3331, + "logp_transplant_std": 2.5627 + }, + "34": { + "recovery_pct": 97.96, + "logp_transplant_mean": -6.1655, + "logp_transplant_std": 2.6042 + }, + "35": { + "recovery_pct": 100.19, + "logp_transplant_mean": -6.0783, + "logp_transplant_std": 2.6289 + } + }, + "L_crit_90": 33, + "L_crit_99": 35, + "alpha_90": 0.9167, + "alpha_99": 0.9722, + "recovery_at_Lcrit": 0.9368, + "n_layers": 36, + "d_model": 1280, + "model": "gpt2-large", + "seed": 7, + "runtime_seconds": 511.0 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2-medium_seed123.json b/data/e1_h3/gpt2-medium_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..5c8b07f5bf07cb609b6a84c586617a6c5d5c0445 --- /dev/null +++ b/data/e1_h3/gpt2-medium_seed123.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -5.2103, + "std": 2.2844 + }, + "baseline_logp_d1000": { + "mean": -9.771, + "std": 1.4826 + }, + "logp_gap": 4.5607, + "layer_sweep": { + "0": { + "recovery_pct": 4.42, + "logp_transplant_mean": -9.5693, + "logp_transplant_std": 1.496 + }, + "1": { + "recovery_pct": -3.02, + "logp_transplant_mean": -9.9086, + "logp_transplant_std": 1.5889 + }, + "2": { + "recovery_pct": -25.17, + "logp_transplant_mean": -10.9191, + "logp_transplant_std": 1.3695 + }, + "3": { + "recovery_pct": -33.9, + "logp_transplant_mean": -11.3169, + "logp_transplant_std": 1.2852 + }, + "4": { + "recovery_pct": -31.03, + "logp_transplant_mean": -11.1861, + "logp_transplant_std": 1.3267 + }, + "5": { + "recovery_pct": -29.3, + "logp_transplant_mean": -11.1075, + "logp_transplant_std": 1.3658 + }, + "6": { + "recovery_pct": -22.54, + "logp_transplant_mean": -10.7992, + "logp_transplant_std": 1.4874 + }, + "7": { + "recovery_pct": -13.99, + "logp_transplant_mean": -10.4089, + "logp_transplant_std": 1.5826 + }, + "8": { + "recovery_pct": -9.29, + "logp_transplant_mean": -10.1947, + "logp_transplant_std": 1.5422 + }, + "9": { + "recovery_pct": -3.22, + "logp_transplant_mean": -9.9177, + "logp_transplant_std": 1.535 + }, + "10": { + "recovery_pct": 0.6, + "logp_transplant_mean": -9.7436, + "logp_transplant_std": 1.5349 + }, + "11": { + "recovery_pct": 12.49, + "logp_transplant_mean": -9.2012, + "logp_transplant_std": 1.6757 + }, + "12": { + "recovery_pct": 26.63, + "logp_transplant_mean": -8.5565, + "logp_transplant_std": 1.7861 + }, + "13": { + "recovery_pct": 22.5, + "logp_transplant_mean": -8.7449, + "logp_transplant_std": 1.6951 + }, + "14": { + "recovery_pct": 24.96, + "logp_transplant_mean": -8.6329, + "logp_transplant_std": 1.7565 + }, + "15": { + "recovery_pct": 25.81, + "logp_transplant_mean": -8.594, + "logp_transplant_std": 1.751 + }, + "16": { + "recovery_pct": 26.06, + "logp_transplant_mean": -8.5825, + "logp_transplant_std": 1.6827 + }, + "17": { + "recovery_pct": 32.32, + "logp_transplant_mean": -8.2969, + "logp_transplant_std": 1.7444 + }, + "18": { + "recovery_pct": 33.71, + "logp_transplant_mean": -8.2338, + "logp_transplant_std": 1.6507 + }, + "19": { + "recovery_pct": 42.52, + "logp_transplant_mean": -7.8318, + "logp_transplant_std": 1.7204 + }, + "20": { + "recovery_pct": 54.08, + "logp_transplant_mean": -7.3047, + "logp_transplant_std": 1.8375 + }, + "21": { + "recovery_pct": 67.4, + "logp_transplant_mean": -6.6972, + "logp_transplant_std": 1.9531 + }, + "22": { + "recovery_pct": 82.1, + "logp_transplant_mean": -6.0265, + "logp_transplant_std": 2.1353 + }, + "23": { + "recovery_pct": 92.95, + "logp_transplant_mean": -5.532, + "logp_transplant_std": 2.1923 + } + }, + "L_crit_90": 23, + "L_crit_99": null, + "alpha_90": 0.9583, + "alpha_99": null, + "recovery_at_Lcrit": 0.9295, + "n_layers": 24, + "d_model": 1024, + "model": "gpt2-medium", + "seed": 123, + "runtime_seconds": 176.0 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2-medium_seed42.json b/data/e1_h3/gpt2-medium_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..953ade4a290a703eed3fd04070543e902ad15243 --- /dev/null +++ b/data/e1_h3/gpt2-medium_seed42.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -5.4631, + "std": 2.8925 + }, + "baseline_logp_d1000": { + "mean": -9.7463, + "std": 1.8955 + }, + "logp_gap": 4.2833, + "layer_sweep": { + "0": { + "recovery_pct": 5.56, + "logp_transplant_mean": -9.508, + "logp_transplant_std": 1.9858 + }, + "1": { + "recovery_pct": -1.42, + "logp_transplant_mean": -9.8073, + "logp_transplant_std": 1.8966 + }, + "2": { + "recovery_pct": -18.65, + "logp_transplant_mean": -10.545, + "logp_transplant_std": 1.3962 + }, + "3": { + "recovery_pct": -29.15, + "logp_transplant_mean": -10.9947, + "logp_transplant_std": 1.3252 + }, + "4": { + "recovery_pct": -25.53, + "logp_transplant_mean": -10.8399, + "logp_transplant_std": 1.2819 + }, + "5": { + "recovery_pct": -23.65, + "logp_transplant_mean": -10.7595, + "logp_transplant_std": 1.3296 + }, + "6": { + "recovery_pct": -18.07, + "logp_transplant_mean": -10.5205, + "logp_transplant_std": 1.4416 + }, + "7": { + "recovery_pct": -8.39, + "logp_transplant_mean": -10.1058, + "logp_transplant_std": 1.7581 + }, + "8": { + "recovery_pct": -3.24, + "logp_transplant_mean": -9.8849, + "logp_transplant_std": 1.8278 + }, + "9": { + "recovery_pct": 1.24, + "logp_transplant_mean": -9.6934, + "logp_transplant_std": 1.9144 + }, + "10": { + "recovery_pct": 3.77, + "logp_transplant_mean": -9.5847, + "logp_transplant_std": 1.8993 + }, + "11": { + "recovery_pct": 15.89, + "logp_transplant_mean": -9.0657, + "logp_transplant_std": 2.1398 + }, + "12": { + "recovery_pct": 28.5, + "logp_transplant_mean": -8.5257, + "logp_transplant_std": 2.2662 + }, + "13": { + "recovery_pct": 24.39, + "logp_transplant_mean": -8.7018, + "logp_transplant_std": 2.0727 + }, + "14": { + "recovery_pct": 25.1, + "logp_transplant_mean": -8.6712, + "logp_transplant_std": 2.1552 + }, + "15": { + "recovery_pct": 23.91, + "logp_transplant_mean": -8.7221, + "logp_transplant_std": 2.0806 + }, + "16": { + "recovery_pct": 23.78, + "logp_transplant_mean": -8.7276, + "logp_transplant_std": 2.117 + }, + "17": { + "recovery_pct": 28.57, + "logp_transplant_mean": -8.5225, + "logp_transplant_std": 2.1925 + }, + "18": { + "recovery_pct": 31.81, + "logp_transplant_mean": -8.3837, + "logp_transplant_std": 2.1712 + }, + "19": { + "recovery_pct": 40.11, + "logp_transplant_mean": -8.0282, + "logp_transplant_std": 2.3 + }, + "20": { + "recovery_pct": 52.07, + "logp_transplant_mean": -7.5162, + "logp_transplant_std": 2.4831 + }, + "21": { + "recovery_pct": 67.74, + "logp_transplant_mean": -6.8449, + "logp_transplant_std": 2.7111 + }, + "22": { + "recovery_pct": 82.6, + "logp_transplant_mean": -6.2084, + "logp_transplant_std": 2.8555 + }, + "23": { + "recovery_pct": 93.2, + "logp_transplant_mean": -5.7542, + "logp_transplant_std": 2.8334 + } + }, + "L_crit_90": 23, + "L_crit_99": null, + "alpha_90": 0.9583, + "alpha_99": null, + "recovery_at_Lcrit": 0.932, + "n_layers": 24, + "d_model": 1024, + "model": "gpt2-medium", + "seed": 42, + "runtime_seconds": 176.4 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2-medium_seed7.json b/data/e1_h3/gpt2-medium_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..3d4e7b32de6180e70d7b259cc518d457335428e2 --- /dev/null +++ b/data/e1_h3/gpt2-medium_seed7.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -5.9067, + "std": 3.0409 + }, + "baseline_logp_d1000": { + "mean": -9.7241, + "std": 2.1412 + }, + "logp_gap": 3.8174, + "layer_sweep": { + "0": { + "recovery_pct": 6.04, + "logp_transplant_mean": -9.4936, + "logp_transplant_std": 2.2336 + }, + "1": { + "recovery_pct": -0.09, + "logp_transplant_mean": -9.7276, + "logp_transplant_std": 2.0944 + }, + "2": { + "recovery_pct": -21.15, + "logp_transplant_mean": -10.5316, + "logp_transplant_std": 1.3673 + }, + "3": { + "recovery_pct": -32.39, + "logp_transplant_mean": -10.9607, + "logp_transplant_std": 1.1423 + }, + "4": { + "recovery_pct": -30.12, + "logp_transplant_mean": -10.8737, + "logp_transplant_std": 1.1727 + }, + "5": { + "recovery_pct": -27.56, + "logp_transplant_mean": -10.776, + "logp_transplant_std": 1.2338 + }, + "6": { + "recovery_pct": -19.52, + "logp_transplant_mean": -10.4692, + "logp_transplant_std": 1.4712 + }, + "7": { + "recovery_pct": -9.87, + "logp_transplant_mean": -10.1009, + "logp_transplant_std": 1.5825 + }, + "8": { + "recovery_pct": -6.28, + "logp_transplant_mean": -9.964, + "logp_transplant_std": 1.5413 + }, + "9": { + "recovery_pct": -2.57, + "logp_transplant_mean": -9.8221, + "logp_transplant_std": 1.6155 + }, + "10": { + "recovery_pct": 1.38, + "logp_transplant_mean": -9.6713, + "logp_transplant_std": 1.6224 + }, + "11": { + "recovery_pct": 11.72, + "logp_transplant_mean": -9.2767, + "logp_transplant_std": 1.9273 + }, + "12": { + "recovery_pct": 24.03, + "logp_transplant_mean": -8.8066, + "logp_transplant_std": 2.0984 + }, + "13": { + "recovery_pct": 19.1, + "logp_transplant_mean": -8.995, + "logp_transplant_std": 1.8581 + }, + "14": { + "recovery_pct": 21.76, + "logp_transplant_mean": -8.8933, + "logp_transplant_std": 1.9676 + }, + "15": { + "recovery_pct": 23.69, + "logp_transplant_mean": -8.8197, + "logp_transplant_std": 1.9113 + }, + "16": { + "recovery_pct": 24.05, + "logp_transplant_mean": -8.8061, + "logp_transplant_std": 1.9449 + }, + "17": { + "recovery_pct": 30.61, + "logp_transplant_mean": -8.5558, + "logp_transplant_std": 2.0788 + }, + "18": { + "recovery_pct": 33.19, + "logp_transplant_mean": -8.457, + "logp_transplant_std": 2.0042 + }, + "19": { + "recovery_pct": 43.4, + "logp_transplant_mean": -8.0674, + "logp_transplant_std": 2.2228 + }, + "20": { + "recovery_pct": 55.8, + "logp_transplant_mean": -7.5939, + "logp_transplant_std": 2.526 + }, + "21": { + "recovery_pct": 68.52, + "logp_transplant_mean": -7.1084, + "logp_transplant_std": 2.7443 + }, + "22": { + "recovery_pct": 81.3, + "logp_transplant_mean": -6.6206, + "logp_transplant_std": 2.9209 + }, + "23": { + "recovery_pct": 94.51, + "logp_transplant_mean": -6.1163, + "logp_transplant_std": 2.9301 + } + }, + "L_crit_90": 23, + "L_crit_99": null, + "alpha_90": 0.9583, + "alpha_99": null, + "recovery_at_Lcrit": 0.9451, + "n_layers": 24, + "d_model": 1024, + "model": "gpt2-medium", + "seed": 7, + "runtime_seconds": 175.7 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2-xl_seed123.json b/data/e1_h3/gpt2-xl_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..c854a35df7118dc6c408d49a6b64623cd80f2aac --- /dev/null +++ b/data/e1_h3/gpt2-xl_seed123.json @@ -0,0 +1,271 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -6.1068, + "std": 2.589 + }, + "baseline_logp_d1000": { + "mean": -10.7776, + "std": 1.6556 + }, + "logp_gap": 4.6708, + "layer_sweep": { + "0": { + "recovery_pct": -0.74, + "logp_transplant_mean": -10.812, + "logp_transplant_std": 1.596 + }, + "1": { + "recovery_pct": -2.23, + "logp_transplant_mean": -10.8816, + "logp_transplant_std": 1.5664 + }, + "2": { + "recovery_pct": -1.81, + "logp_transplant_mean": -10.8623, + "logp_transplant_std": 1.5771 + }, + "3": { + "recovery_pct": 0.6, + "logp_transplant_mean": -10.7495, + "logp_transplant_std": 1.7063 + }, + "4": { + "recovery_pct": 8.48, + "logp_transplant_mean": -10.3813, + "logp_transplant_std": 2.002 + }, + "5": { + "recovery_pct": 4.55, + "logp_transplant_mean": -10.565, + "logp_transplant_std": 1.8072 + }, + "6": { + "recovery_pct": 6.52, + "logp_transplant_mean": -10.4731, + "logp_transplant_std": 1.8261 + }, + "7": { + "recovery_pct": 6.53, + "logp_transplant_mean": -10.4727, + "logp_transplant_std": 1.8417 + }, + "8": { + "recovery_pct": 2.93, + "logp_transplant_mean": -10.6406, + "logp_transplant_std": 1.7092 + }, + "9": { + "recovery_pct": 2.08, + "logp_transplant_mean": -10.6803, + "logp_transplant_std": 1.6812 + }, + "10": { + "recovery_pct": 1.87, + "logp_transplant_mean": -10.6901, + "logp_transplant_std": 1.6302 + }, + "11": { + "recovery_pct": 2.56, + "logp_transplant_mean": -10.6581, + "logp_transplant_std": 1.6429 + }, + "12": { + "recovery_pct": -1.15, + "logp_transplant_mean": -10.8315, + "logp_transplant_std": 1.5511 + }, + "13": { + "recovery_pct": -0.81, + "logp_transplant_mean": -10.8157, + "logp_transplant_std": 1.5128 + }, + "14": { + "recovery_pct": 1.63, + "logp_transplant_mean": -10.7017, + "logp_transplant_std": 1.5034 + }, + "15": { + "recovery_pct": 3.61, + "logp_transplant_mean": -10.6091, + "logp_transplant_std": 1.4998 + }, + "16": { + "recovery_pct": 8.83, + "logp_transplant_mean": -10.3652, + "logp_transplant_std": 1.589 + }, + "17": { + "recovery_pct": 8.39, + "logp_transplant_mean": -10.386, + "logp_transplant_std": 1.6282 + }, + "18": { + "recovery_pct": 7.6, + "logp_transplant_mean": -10.4229, + "logp_transplant_std": 1.5825 + }, + "19": { + "recovery_pct": 15.19, + "logp_transplant_mean": -10.0684, + "logp_transplant_std": 1.6868 + }, + "20": { + "recovery_pct": 21.97, + "logp_transplant_mean": -9.7514, + "logp_transplant_std": 1.7709 + }, + "21": { + "recovery_pct": 29.16, + "logp_transplant_mean": -9.4156, + "logp_transplant_std": 1.8673 + }, + "22": { + "recovery_pct": 36.89, + "logp_transplant_mean": -9.0546, + "logp_transplant_std": 1.9927 + }, + "23": { + "recovery_pct": 39.04, + "logp_transplant_mean": -8.9541, + "logp_transplant_std": 2.0406 + }, + "24": { + "recovery_pct": 40.02, + "logp_transplant_mean": -8.9082, + "logp_transplant_std": 2.009 + }, + "25": { + "recovery_pct": 47.9, + "logp_transplant_mean": -8.5403, + "logp_transplant_std": 2.1029 + }, + "26": { + "recovery_pct": 50.3, + "logp_transplant_mean": -8.4283, + "logp_transplant_std": 2.1486 + }, + "27": { + "recovery_pct": 49.85, + "logp_transplant_mean": -8.449, + "logp_transplant_std": 2.113 + }, + "28": { + "recovery_pct": 49.81, + "logp_transplant_mean": -8.4513, + "logp_transplant_std": 2.1829 + }, + "29": { + "recovery_pct": 53.53, + "logp_transplant_mean": -8.2772, + "logp_transplant_std": 2.2809 + }, + "30": { + "recovery_pct": 54.81, + "logp_transplant_mean": -8.2176, + "logp_transplant_std": 2.1936 + }, + "31": { + "recovery_pct": 52.54, + "logp_transplant_mean": -8.3237, + "logp_transplant_std": 2.0964 + }, + "32": { + "recovery_pct": 56.03, + "logp_transplant_mean": -8.1607, + "logp_transplant_std": 2.1807 + }, + "33": { + "recovery_pct": 56.67, + "logp_transplant_mean": -8.1305, + "logp_transplant_std": 2.1518 + }, + "34": { + "recovery_pct": 61.18, + "logp_transplant_mean": -7.9201, + "logp_transplant_std": 2.2444 + }, + "35": { + "recovery_pct": 64.91, + "logp_transplant_mean": -7.7458, + "logp_transplant_std": 2.2279 + }, + "36": { + "recovery_pct": 69.54, + "logp_transplant_mean": -7.5297, + "logp_transplant_std": 2.2359 + }, + "37": { + "recovery_pct": 71.35, + "logp_transplant_mean": -7.4449, + "logp_transplant_std": 2.2532 + }, + "38": { + "recovery_pct": 74.03, + "logp_transplant_mean": -7.3196, + "logp_transplant_std": 2.2638 + }, + "39": { + "recovery_pct": 78.54, + "logp_transplant_mean": -7.1094, + "logp_transplant_std": 2.3269 + }, + "40": { + "recovery_pct": 81.49, + "logp_transplant_mean": -6.9715, + "logp_transplant_std": 2.3364 + }, + "41": { + "recovery_pct": 86.23, + "logp_transplant_mean": -6.7502, + "logp_transplant_std": 2.3837 + }, + "42": { + "recovery_pct": 88.99, + "logp_transplant_mean": -6.6209, + "logp_transplant_std": 2.4614 + }, + "43": { + "recovery_pct": 92.43, + "logp_transplant_mean": -6.4604, + "logp_transplant_std": 2.5157 + }, + "44": { + "recovery_pct": 95.49, + "logp_transplant_mean": -6.3174, + "logp_transplant_std": 2.5227 + }, + "45": { + "recovery_pct": 98.42, + "logp_transplant_mean": -6.1808, + "logp_transplant_std": 2.5885 + }, + "46": { + "recovery_pct": 99.97, + "logp_transplant_mean": -6.1084, + "logp_transplant_std": 2.5872 + }, + "47": { + "recovery_pct": 100.54, + "logp_transplant_mean": -6.0817, + "logp_transplant_std": 2.6 + } + }, + "L_crit_90": 43, + "L_crit_99": 46, + "alpha_90": 0.8958, + "alpha_99": 0.9583, + "recovery_at_Lcrit": 0.9243, + "n_layers": 48, + "d_model": 1600, + "model": "gpt2-xl", + "seed": 123, + "runtime_seconds": 1241.2 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2-xl_seed42.json b/data/e1_h3/gpt2-xl_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..c860fbc0588ad54725f531d9504130d3b1f78457 --- /dev/null +++ b/data/e1_h3/gpt2-xl_seed42.json @@ -0,0 +1,271 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -6.0101, + "std": 2.7684 + }, + "baseline_logp_d1000": { + "mean": -10.7722, + "std": 1.8118 + }, + "logp_gap": 4.762, + "layer_sweep": { + "0": { + "recovery_pct": -0.67, + "logp_transplant_mean": -10.8041, + "logp_transplant_std": 1.7484 + }, + "1": { + "recovery_pct": -1.6, + "logp_transplant_mean": -10.8484, + "logp_transplant_std": 1.7243 + }, + "2": { + "recovery_pct": -0.62, + "logp_transplant_mean": -10.8014, + "logp_transplant_std": 1.7603 + }, + "3": { + "recovery_pct": 1.54, + "logp_transplant_mean": -10.6988, + "logp_transplant_std": 1.7851 + }, + "4": { + "recovery_pct": 5.86, + "logp_transplant_mean": -10.4929, + "logp_transplant_std": 1.9665 + }, + "5": { + "recovery_pct": 3.69, + "logp_transplant_mean": -10.5964, + "logp_transplant_std": 1.8037 + }, + "6": { + "recovery_pct": 6.1, + "logp_transplant_mean": -10.4815, + "logp_transplant_std": 1.8284 + }, + "7": { + "recovery_pct": 5.72, + "logp_transplant_mean": -10.4996, + "logp_transplant_std": 1.8839 + }, + "8": { + "recovery_pct": 4.45, + "logp_transplant_mean": -10.5603, + "logp_transplant_std": 1.769 + }, + "9": { + "recovery_pct": 3.62, + "logp_transplant_mean": -10.5999, + "logp_transplant_std": 1.7496 + }, + "10": { + "recovery_pct": 3.29, + "logp_transplant_mean": -10.6155, + "logp_transplant_std": 1.6963 + }, + "11": { + "recovery_pct": 4.07, + "logp_transplant_mean": -10.5785, + "logp_transplant_std": 1.6989 + }, + "12": { + "recovery_pct": 2.17, + "logp_transplant_mean": -10.6686, + "logp_transplant_std": 1.6391 + }, + "13": { + "recovery_pct": 2.84, + "logp_transplant_mean": -10.637, + "logp_transplant_std": 1.6874 + }, + "14": { + "recovery_pct": 4.28, + "logp_transplant_mean": -10.5683, + "logp_transplant_std": 1.7259 + }, + "15": { + "recovery_pct": 5.38, + "logp_transplant_mean": -10.5161, + "logp_transplant_std": 1.7665 + }, + "16": { + "recovery_pct": 9.68, + "logp_transplant_mean": -10.3111, + "logp_transplant_std": 1.9126 + }, + "17": { + "recovery_pct": 9.41, + "logp_transplant_mean": -10.3239, + "logp_transplant_std": 1.9505 + }, + "18": { + "recovery_pct": 8.21, + "logp_transplant_mean": -10.3814, + "logp_transplant_std": 1.9049 + }, + "19": { + "recovery_pct": 14.67, + "logp_transplant_mean": -10.0734, + "logp_transplant_std": 1.9501 + }, + "20": { + "recovery_pct": 21.1, + "logp_transplant_mean": -9.7676, + "logp_transplant_std": 2.1252 + }, + "21": { + "recovery_pct": 27.27, + "logp_transplant_mean": -9.4733, + "logp_transplant_std": 2.2339 + }, + "22": { + "recovery_pct": 34.0, + "logp_transplant_mean": -9.1532, + "logp_transplant_std": 2.3275 + }, + "23": { + "recovery_pct": 36.19, + "logp_transplant_mean": -9.0486, + "logp_transplant_std": 2.382 + }, + "24": { + "recovery_pct": 37.71, + "logp_transplant_mean": -8.9764, + "logp_transplant_std": 2.3272 + }, + "25": { + "recovery_pct": 45.5, + "logp_transplant_mean": -8.6056, + "logp_transplant_std": 2.5183 + }, + "26": { + "recovery_pct": 46.71, + "logp_transplant_mean": -8.5477, + "logp_transplant_std": 2.5544 + }, + "27": { + "recovery_pct": 45.42, + "logp_transplant_mean": -8.6094, + "logp_transplant_std": 2.464 + }, + "28": { + "recovery_pct": 45.21, + "logp_transplant_mean": -8.6192, + "logp_transplant_std": 2.4324 + }, + "29": { + "recovery_pct": 47.78, + "logp_transplant_mean": -8.4967, + "logp_transplant_std": 2.4581 + }, + "30": { + "recovery_pct": 47.68, + "logp_transplant_mean": -8.5017, + "logp_transplant_std": 2.3658 + }, + "31": { + "recovery_pct": 46.1, + "logp_transplant_mean": -8.5767, + "logp_transplant_std": 2.2636 + }, + "32": { + "recovery_pct": 50.55, + "logp_transplant_mean": -8.3651, + "logp_transplant_std": 2.3411 + }, + "33": { + "recovery_pct": 52.65, + "logp_transplant_mean": -8.2651, + "logp_transplant_std": 2.3238 + }, + "34": { + "recovery_pct": 57.13, + "logp_transplant_mean": -8.0517, + "logp_transplant_std": 2.3993 + }, + "35": { + "recovery_pct": 61.2, + "logp_transplant_mean": -7.8579, + "logp_transplant_std": 2.4713 + }, + "36": { + "recovery_pct": 66.57, + "logp_transplant_mean": -7.602, + "logp_transplant_std": 2.5163 + }, + "37": { + "recovery_pct": 68.86, + "logp_transplant_mean": -7.4932, + "logp_transplant_std": 2.544 + }, + "38": { + "recovery_pct": 72.04, + "logp_transplant_mean": -7.3417, + "logp_transplant_std": 2.5854 + }, + "39": { + "recovery_pct": 77.66, + "logp_transplant_mean": -7.0738, + "logp_transplant_std": 2.66 + }, + "40": { + "recovery_pct": 79.9, + "logp_transplant_mean": -6.9673, + "logp_transplant_std": 2.6734 + }, + "41": { + "recovery_pct": 84.68, + "logp_transplant_mean": -6.7399, + "logp_transplant_std": 2.7111 + }, + "42": { + "recovery_pct": 87.7, + "logp_transplant_mean": -6.596, + "logp_transplant_std": 2.7066 + }, + "43": { + "recovery_pct": 91.05, + "logp_transplant_mean": -6.4362, + "logp_transplant_std": 2.7429 + }, + "44": { + "recovery_pct": 94.6, + "logp_transplant_mean": -6.2671, + "logp_transplant_std": 2.7547 + }, + "45": { + "recovery_pct": 98.51, + "logp_transplant_mean": -6.0812, + "logp_transplant_std": 2.799 + }, + "46": { + "recovery_pct": 99.8, + "logp_transplant_mean": -6.0198, + "logp_transplant_std": 2.8075 + }, + "47": { + "recovery_pct": 100.52, + "logp_transplant_mean": -5.9854, + "logp_transplant_std": 2.8109 + } + }, + "L_crit_90": 43, + "L_crit_99": 46, + "alpha_90": 0.8958, + "alpha_99": 0.9583, + "recovery_at_Lcrit": 0.9105, + "n_layers": 48, + "d_model": 1600, + "model": "gpt2-xl", + "seed": 42, + "runtime_seconds": 1237.8 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2-xl_seed7.json b/data/e1_h3/gpt2-xl_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..b11fa3a7a1bf2b6f758d5c2ad23bc21a1a703574 --- /dev/null +++ b/data/e1_h3/gpt2-xl_seed7.json @@ -0,0 +1,271 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -6.1311, + "std": 2.8704 + }, + "baseline_logp_d1000": { + "mean": -10.6676, + "std": 1.8468 + }, + "logp_gap": 4.5365, + "layer_sweep": { + "0": { + "recovery_pct": -0.51, + "logp_transplant_mean": -10.6907, + "logp_transplant_std": 1.8202 + }, + "1": { + "recovery_pct": -1.59, + "logp_transplant_mean": -10.7395, + "logp_transplant_std": 1.8135 + }, + "2": { + "recovery_pct": -0.15, + "logp_transplant_mean": -10.6743, + "logp_transplant_std": 1.8851 + }, + "3": { + "recovery_pct": 3.68, + "logp_transplant_mean": -10.5005, + "logp_transplant_std": 2.0913 + }, + "4": { + "recovery_pct": 8.38, + "logp_transplant_mean": -10.2873, + "logp_transplant_std": 2.2274 + }, + "5": { + "recovery_pct": 5.83, + "logp_transplant_mean": -10.4031, + "logp_transplant_std": 2.1211 + }, + "6": { + "recovery_pct": 6.91, + "logp_transplant_mean": -10.3542, + "logp_transplant_std": 2.1677 + }, + "7": { + "recovery_pct": 5.58, + "logp_transplant_mean": -10.4145, + "logp_transplant_std": 2.1666 + }, + "8": { + "recovery_pct": 3.2, + "logp_transplant_mean": -10.5225, + "logp_transplant_std": 1.9956 + }, + "9": { + "recovery_pct": 2.34, + "logp_transplant_mean": -10.5616, + "logp_transplant_std": 1.9494 + }, + "10": { + "recovery_pct": 1.73, + "logp_transplant_mean": -10.5892, + "logp_transplant_std": 1.8673 + }, + "11": { + "recovery_pct": 2.45, + "logp_transplant_mean": -10.5565, + "logp_transplant_std": 1.8772 + }, + "12": { + "recovery_pct": -1.58, + "logp_transplant_mean": -10.7393, + "logp_transplant_std": 1.7629 + }, + "13": { + "recovery_pct": -1.05, + "logp_transplant_mean": -10.7154, + "logp_transplant_std": 1.769 + }, + "14": { + "recovery_pct": 2.11, + "logp_transplant_mean": -10.572, + "logp_transplant_std": 1.8271 + }, + "15": { + "recovery_pct": 4.28, + "logp_transplant_mean": -10.4733, + "logp_transplant_std": 1.88 + }, + "16": { + "recovery_pct": 9.44, + "logp_transplant_mean": -10.2393, + "logp_transplant_std": 2.0824 + }, + "17": { + "recovery_pct": 9.23, + "logp_transplant_mean": -10.2489, + "logp_transplant_std": 2.1382 + }, + "18": { + "recovery_pct": 9.06, + "logp_transplant_mean": -10.2567, + "logp_transplant_std": 2.1072 + }, + "19": { + "recovery_pct": 17.71, + "logp_transplant_mean": -9.8643, + "logp_transplant_std": 2.2154 + }, + "20": { + "recovery_pct": 22.84, + "logp_transplant_mean": -9.6317, + "logp_transplant_std": 2.3405 + }, + "21": { + "recovery_pct": 28.15, + "logp_transplant_mean": -9.3908, + "logp_transplant_std": 2.4645 + }, + "22": { + "recovery_pct": 34.35, + "logp_transplant_mean": -9.1092, + "logp_transplant_std": 2.5233 + }, + "23": { + "recovery_pct": 34.99, + "logp_transplant_mean": -9.0802, + "logp_transplant_std": 2.5107 + }, + "24": { + "recovery_pct": 35.89, + "logp_transplant_mean": -9.0393, + "logp_transplant_std": 2.4775 + }, + "25": { + "recovery_pct": 43.11, + "logp_transplant_mean": -8.712, + "logp_transplant_std": 2.6621 + }, + "26": { + "recovery_pct": 45.19, + "logp_transplant_mean": -8.6175, + "logp_transplant_std": 2.6992 + }, + "27": { + "recovery_pct": 44.75, + "logp_transplant_mean": -8.6374, + "logp_transplant_std": 2.6395 + }, + "28": { + "recovery_pct": 44.99, + "logp_transplant_mean": -8.6268, + "logp_transplant_std": 2.6259 + }, + "29": { + "recovery_pct": 49.18, + "logp_transplant_mean": -8.4365, + "logp_transplant_std": 2.7066 + }, + "30": { + "recovery_pct": 49.9, + "logp_transplant_mean": -8.404, + "logp_transplant_std": 2.6381 + }, + "31": { + "recovery_pct": 48.1, + "logp_transplant_mean": -8.4854, + "logp_transplant_std": 2.5323 + }, + "32": { + "recovery_pct": 52.8, + "logp_transplant_mean": -8.2725, + "logp_transplant_std": 2.6267 + }, + "33": { + "recovery_pct": 53.24, + "logp_transplant_mean": -8.2523, + "logp_transplant_std": 2.5726 + }, + "34": { + "recovery_pct": 56.57, + "logp_transplant_mean": -8.1011, + "logp_transplant_std": 2.5822 + }, + "35": { + "recovery_pct": 62.14, + "logp_transplant_mean": -7.8488, + "logp_transplant_std": 2.6612 + }, + "36": { + "recovery_pct": 67.02, + "logp_transplant_mean": -7.627, + "logp_transplant_std": 2.687 + }, + "37": { + "recovery_pct": 68.45, + "logp_transplant_mean": -7.5624, + "logp_transplant_std": 2.746 + }, + "38": { + "recovery_pct": 73.3, + "logp_transplant_mean": -7.3424, + "logp_transplant_std": 2.7967 + }, + "39": { + "recovery_pct": 78.31, + "logp_transplant_mean": -7.115, + "logp_transplant_std": 2.8228 + }, + "40": { + "recovery_pct": 81.57, + "logp_transplant_mean": -6.9672, + "logp_transplant_std": 2.786 + }, + "41": { + "recovery_pct": 86.57, + "logp_transplant_mean": -6.7404, + "logp_transplant_std": 2.8 + }, + "42": { + "recovery_pct": 89.61, + "logp_transplant_mean": -6.6023, + "logp_transplant_std": 2.8288 + }, + "43": { + "recovery_pct": 93.05, + "logp_transplant_mean": -6.4463, + "logp_transplant_std": 2.8746 + }, + "44": { + "recovery_pct": 96.06, + "logp_transplant_mean": -6.3099, + "logp_transplant_std": 2.8621 + }, + "45": { + "recovery_pct": 98.67, + "logp_transplant_mean": -6.1914, + "logp_transplant_std": 2.8952 + }, + "46": { + "recovery_pct": 100.17, + "logp_transplant_mean": -6.1233, + "logp_transplant_std": 2.8761 + }, + "47": { + "recovery_pct": 100.54, + "logp_transplant_mean": -6.1065, + "logp_transplant_std": 2.8737 + } + }, + "L_crit_90": 43, + "L_crit_99": 46, + "alpha_90": 0.8958, + "alpha_99": 0.9583, + "recovery_at_Lcrit": 0.9305, + "n_layers": 48, + "d_model": 1600, + "model": "gpt2-xl", + "seed": 7, + "runtime_seconds": 1254.7 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2_seed123.json b/data/e1_h3/gpt2_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..108964f28dc05f80a42351dcc33de7e7b1f6034d --- /dev/null +++ b/data/e1_h3/gpt2_seed123.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -4.9983, + "std": 2.0358 + }, + "baseline_logp_d1000": { + "mean": -9.7232, + "std": 1.7204 + }, + "logp_gap": 4.7249, + "layer_sweep": { + "0": { + "recovery_pct": -27.72, + "logp_transplant_mean": -11.0327, + "logp_transplant_std": 1.3037 + }, + "1": { + "recovery_pct": -18.84, + "logp_transplant_mean": -10.6132, + "logp_transplant_std": 1.3901 + }, + "2": { + "recovery_pct": -22.63, + "logp_transplant_mean": -10.7922, + "logp_transplant_std": 1.2785 + }, + "3": { + "recovery_pct": -30.2, + "logp_transplant_mean": -11.1502, + "logp_transplant_std": 1.1638 + }, + "4": { + "recovery_pct": -28.85, + "logp_transplant_mean": -11.0863, + "logp_transplant_std": 1.1604 + }, + "5": { + "recovery_pct": -26.36, + "logp_transplant_mean": -10.9688, + "logp_transplant_std": 1.2083 + }, + "6": { + "recovery_pct": -17.31, + "logp_transplant_mean": -10.5412, + "logp_transplant_std": 1.2158 + }, + "7": { + "recovery_pct": -4.48, + "logp_transplant_mean": -9.9348, + "logp_transplant_std": 1.1707 + }, + "8": { + "recovery_pct": 9.0, + "logp_transplant_mean": -9.2978, + "logp_transplant_std": 1.1846 + }, + "9": { + "recovery_pct": 17.77, + "logp_transplant_mean": -8.8838, + "logp_transplant_std": 1.1965 + }, + "10": { + "recovery_pct": 46.9, + "logp_transplant_mean": -7.5073, + "logp_transplant_std": 1.6259 + }, + "11": { + "recovery_pct": 80.35, + "logp_transplant_mean": -5.9269, + "logp_transplant_std": 1.9082 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 12, + "d_model": 768, + "model": "gpt2", + "seed": 123, + "runtime_seconds": 32.9 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2_seed42.json b/data/e1_h3/gpt2_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..c68c5ad028edda1767f038a6873135c2fdba50b7 --- /dev/null +++ b/data/e1_h3/gpt2_seed42.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -4.863, + "std": 2.3368 + }, + "baseline_logp_d1000": { + "mean": -9.4799, + "std": 1.9428 + }, + "logp_gap": 4.6169, + "layer_sweep": { + "0": { + "recovery_pct": -28.69, + "logp_transplant_mean": -10.8045, + "logp_transplant_std": 1.1674 + }, + "1": { + "recovery_pct": -24.51, + "logp_transplant_mean": -10.6115, + "logp_transplant_std": 1.394 + }, + "2": { + "recovery_pct": -29.58, + "logp_transplant_mean": -10.8457, + "logp_transplant_std": 1.2745 + }, + "3": { + "recovery_pct": -33.39, + "logp_transplant_mean": -11.0214, + "logp_transplant_std": 1.2161 + }, + "4": { + "recovery_pct": -30.8, + "logp_transplant_mean": -10.9018, + "logp_transplant_std": 1.2152 + }, + "5": { + "recovery_pct": -28.74, + "logp_transplant_mean": -10.8068, + "logp_transplant_std": 1.2455 + }, + "6": { + "recovery_pct": -18.03, + "logp_transplant_mean": -10.3125, + "logp_transplant_std": 1.2084 + }, + "7": { + "recovery_pct": -4.27, + "logp_transplant_mean": -9.6769, + "logp_transplant_std": 1.2674 + }, + "8": { + "recovery_pct": 10.25, + "logp_transplant_mean": -9.0066, + "logp_transplant_std": 1.4436 + }, + "9": { + "recovery_pct": 18.5, + "logp_transplant_mean": -8.6255, + "logp_transplant_std": 1.4597 + }, + "10": { + "recovery_pct": 46.12, + "logp_transplant_mean": -7.3507, + "logp_transplant_std": 1.8884 + }, + "11": { + "recovery_pct": 80.28, + "logp_transplant_mean": -5.7733, + "logp_transplant_std": 2.1807 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 12, + "d_model": 768, + "model": "gpt2", + "seed": 42, + "runtime_seconds": 32.9 +} \ No newline at end of file diff --git a/data/e1_h3/gpt2_seed7.json b/data/e1_h3/gpt2_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..ddd68075cf6447697d474616c11b8408a35356cb --- /dev/null +++ b/data/e1_h3/gpt2_seed7.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -5.1622, + "std": 2.4225 + }, + "baseline_logp_d1000": { + "mean": -9.513, + "std": 2.2446 + }, + "logp_gap": 4.3509, + "layer_sweep": { + "0": { + "recovery_pct": -31.02, + "logp_transplant_mean": -10.8627, + "logp_transplant_std": 1.4275 + }, + "1": { + "recovery_pct": -31.11, + "logp_transplant_mean": -10.8667, + "logp_transplant_std": 1.5341 + }, + "2": { + "recovery_pct": -31.84, + "logp_transplant_mean": -10.8983, + "logp_transplant_std": 1.1872 + }, + "3": { + "recovery_pct": -36.91, + "logp_transplant_mean": -11.1188, + "logp_transplant_std": 1.0651 + }, + "4": { + "recovery_pct": -33.9, + "logp_transplant_mean": -10.9879, + "logp_transplant_std": 1.0012 + }, + "5": { + "recovery_pct": -29.9, + "logp_transplant_mean": -10.8137, + "logp_transplant_std": 0.987 + }, + "6": { + "recovery_pct": -19.62, + "logp_transplant_mean": -10.3664, + "logp_transplant_std": 1.0476 + }, + "7": { + "recovery_pct": -6.35, + "logp_transplant_mean": -9.7892, + "logp_transplant_std": 1.2019 + }, + "8": { + "recovery_pct": 7.39, + "logp_transplant_mean": -9.1916, + "logp_transplant_std": 1.4155 + }, + "9": { + "recovery_pct": 17.31, + "logp_transplant_mean": -8.7599, + "logp_transplant_std": 1.4997 + }, + "10": { + "recovery_pct": 45.37, + "logp_transplant_mean": -7.5392, + "logp_transplant_std": 1.9887 + }, + "11": { + "recovery_pct": 79.17, + "logp_transplant_mean": -6.0686, + "logp_transplant_std": 2.1742 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 12, + "d_model": 768, + "model": "gpt2", + "seed": 7, + "runtime_seconds": 32.7 +} \ No newline at end of file diff --git a/data/e1_h3/state-spaces--mamba-2.8b_seed123.json b/data/e1_h3/state-spaces--mamba-2.8b_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..b30b3b05d951ad933740199c89eec26ed37e7ff0 --- /dev/null +++ b/data/e1_h3/state-spaces--mamba-2.8b_seed123.json @@ -0,0 +1,350 @@ +{ + "model_type": "mamba_ssm", + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 25, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -6.5087, + "std": 2.0768 + }, + "baseline_logp_d1000": { + "mean": -11.7802, + "std": 1.0988 + }, + "logp_gap": 5.2716, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -11.7802, + "logp_transplant_std": 1.0988 + }, + "1": { + "recovery_pct": -0.18, + "logp_transplant_mean": -11.7898, + "logp_transplant_std": 1.1569 + }, + "2": { + "recovery_pct": 4.28, + "logp_transplant_mean": -11.5544, + "logp_transplant_std": 1.1592 + }, + "3": { + "recovery_pct": 3.96, + "logp_transplant_mean": -11.5714, + "logp_transplant_std": 1.1857 + }, + "4": { + "recovery_pct": 3.71, + "logp_transplant_mean": -11.5848, + "logp_transplant_std": 1.1628 + }, + "5": { + "recovery_pct": 4.01, + "logp_transplant_mean": -11.5686, + "logp_transplant_std": 1.2155 + }, + "6": { + "recovery_pct": 4.07, + "logp_transplant_mean": -11.5657, + "logp_transplant_std": 1.1764 + }, + "7": { + "recovery_pct": 4.22, + "logp_transplant_mean": -11.5579, + "logp_transplant_std": 1.1926 + }, + "8": { + "recovery_pct": 3.86, + "logp_transplant_mean": -11.5767, + "logp_transplant_std": 1.1623 + }, + "9": { + "recovery_pct": 3.97, + "logp_transplant_mean": -11.571, + "logp_transplant_std": 1.1581 + }, + "10": { + "recovery_pct": 4.03, + "logp_transplant_mean": -11.5679, + "logp_transplant_std": 1.1707 + }, + "11": { + "recovery_pct": 7.93, + "logp_transplant_mean": -11.3621, + "logp_transplant_std": 1.2879 + }, + "12": { + "recovery_pct": 7.24, + "logp_transplant_mean": -11.3985, + "logp_transplant_std": 1.2844 + }, + "13": { + "recovery_pct": 7.22, + "logp_transplant_mean": -11.3995, + "logp_transplant_std": 1.2797 + }, + "14": { + "recovery_pct": 7.0, + "logp_transplant_mean": -11.4112, + "logp_transplant_std": 1.2606 + }, + "15": { + "recovery_pct": 7.11, + "logp_transplant_mean": -11.4054, + "logp_transplant_std": 1.2606 + }, + "16": { + "recovery_pct": 6.51, + "logp_transplant_mean": -11.4371, + "logp_transplant_std": 1.2229 + }, + "17": { + "recovery_pct": 6.77, + "logp_transplant_mean": -11.4235, + "logp_transplant_std": 1.2163 + }, + "18": { + "recovery_pct": 6.26, + "logp_transplant_mean": -11.4501, + "logp_transplant_std": 1.209 + }, + "19": { + "recovery_pct": 7.34, + "logp_transplant_mean": -11.3933, + "logp_transplant_std": 1.2653 + }, + "20": { + "recovery_pct": 7.19, + "logp_transplant_mean": -11.401, + "logp_transplant_std": 1.264 + }, + "21": { + "recovery_pct": 7.37, + "logp_transplant_mean": -11.3919, + "logp_transplant_std": 1.2945 + }, + "22": { + "recovery_pct": 7.25, + "logp_transplant_mean": -11.398, + "logp_transplant_std": 1.3094 + }, + "23": { + "recovery_pct": 7.35, + "logp_transplant_mean": -11.3928, + "logp_transplant_std": 1.3744 + }, + "24": { + "recovery_pct": 8.83, + "logp_transplant_mean": -11.3146, + "logp_transplant_std": 1.5151 + }, + "25": { + "recovery_pct": 9.48, + "logp_transplant_mean": -11.2802, + "logp_transplant_std": 1.5697 + }, + "26": { + "recovery_pct": 9.98, + "logp_transplant_mean": -11.2541, + "logp_transplant_std": 1.5491 + }, + "27": { + "recovery_pct": 10.02, + "logp_transplant_mean": -11.2519, + "logp_transplant_std": 1.5208 + }, + "28": { + "recovery_pct": 9.53, + "logp_transplant_mean": -11.2776, + "logp_transplant_std": 1.5642 + }, + "29": { + "recovery_pct": 12.36, + "logp_transplant_mean": -11.1287, + "logp_transplant_std": 1.6826 + }, + "30": { + "recovery_pct": 8.2, + "logp_transplant_mean": -11.3479, + "logp_transplant_std": 1.4459 + }, + "31": { + "recovery_pct": 26.2, + "logp_transplant_mean": -10.3988, + "logp_transplant_std": 1.8647 + }, + "32": { + "recovery_pct": 24.93, + "logp_transplant_mean": -10.4661, + "logp_transplant_std": 1.8088 + }, + "33": { + "recovery_pct": 25.86, + "logp_transplant_mean": -10.4168, + "logp_transplant_std": 1.8057 + }, + "34": { + "recovery_pct": 29.76, + "logp_transplant_mean": -10.2111, + "logp_transplant_std": 1.8414 + }, + "35": { + "recovery_pct": 30.15, + "logp_transplant_mean": -10.1907, + "logp_transplant_std": 1.8758 + }, + "36": { + "recovery_pct": 32.33, + "logp_transplant_mean": -10.0758, + "logp_transplant_std": 1.9394 + }, + "37": { + "recovery_pct": 33.41, + "logp_transplant_mean": -10.0191, + "logp_transplant_std": 1.933 + }, + "38": { + "recovery_pct": 33.89, + "logp_transplant_mean": -9.9936, + "logp_transplant_std": 1.9287 + }, + "39": { + "recovery_pct": 34.85, + "logp_transplant_mean": -9.9429, + "logp_transplant_std": 1.9033 + }, + "40": { + "recovery_pct": 35.38, + "logp_transplant_mean": -9.915, + "logp_transplant_std": 1.9584 + }, + "41": { + "recovery_pct": 33.84, + "logp_transplant_mean": -9.9961, + "logp_transplant_std": 1.8755 + }, + "42": { + "recovery_pct": 76.42, + "logp_transplant_mean": -7.7516, + "logp_transplant_std": 2.2565 + }, + "43": { + "recovery_pct": 78.8, + "logp_transplant_mean": -7.6264, + "logp_transplant_std": 2.3226 + }, + "44": { + "recovery_pct": 76.55, + "logp_transplant_mean": -7.7451, + "logp_transplant_std": 2.2208 + }, + "45": { + "recovery_pct": 108.26, + "logp_transplant_mean": -6.0735, + "logp_transplant_std": 2.0031 + }, + "46": { + "recovery_pct": 107.72, + "logp_transplant_mean": -6.1015, + "logp_transplant_std": 2.0121 + }, + "47": { + "recovery_pct": 106.7, + "logp_transplant_mean": -6.1557, + "logp_transplant_std": 2.0251 + }, + "48": { + "recovery_pct": 106.67, + "logp_transplant_mean": -6.1569, + "logp_transplant_std": 2.0271 + }, + "49": { + "recovery_pct": 106.41, + "logp_transplant_mean": -6.1706, + "logp_transplant_std": 2.0596 + }, + "50": { + "recovery_pct": 106.28, + "logp_transplant_mean": -6.1775, + "logp_transplant_std": 2.0684 + }, + "51": { + "recovery_pct": 105.52, + "logp_transplant_mean": -6.2176, + "logp_transplant_std": 2.0735 + }, + "52": { + "recovery_pct": 104.77, + "logp_transplant_mean": -6.2573, + "logp_transplant_std": 2.1065 + }, + "53": { + "recovery_pct": 105.33, + "logp_transplant_mean": -6.2278, + "logp_transplant_std": 2.1274 + }, + "54": { + "recovery_pct": 103.64, + "logp_transplant_mean": -6.3166, + "logp_transplant_std": 2.1361 + }, + "55": { + "recovery_pct": 102.62, + "logp_transplant_mean": -6.3704, + "logp_transplant_std": 2.1232 + }, + "56": { + "recovery_pct": 102.16, + "logp_transplant_mean": -6.3951, + "logp_transplant_std": 2.1115 + }, + "57": { + "recovery_pct": 102.05, + "logp_transplant_mean": -6.4004, + "logp_transplant_std": 2.1216 + }, + "58": { + "recovery_pct": 101.97, + "logp_transplant_mean": -6.4048, + "logp_transplant_std": 2.0898 + }, + "59": { + "recovery_pct": 101.38, + "logp_transplant_mean": -6.4357, + "logp_transplant_std": 2.095 + }, + "60": { + "recovery_pct": 101.39, + "logp_transplant_mean": -6.4352, + "logp_transplant_std": 2.0981 + }, + "61": { + "recovery_pct": 101.21, + "logp_transplant_mean": -6.4447, + "logp_transplant_std": 2.0842 + }, + "62": { + "recovery_pct": 100.34, + "logp_transplant_mean": -6.4907, + "logp_transplant_std": 2.0915 + }, + "63": { + "recovery_pct": 100.91, + "logp_transplant_mean": -6.4606, + "logp_transplant_std": 2.0897 + } + }, + "L_crit_90": 45, + "L_crit_99": 45, + "alpha_90": 0.7031, + "alpha_99": 0.7031, + "n_layers": 64, + "d_model": 2560, + "seed": 123, + "runtime_seconds": 8278.2 +} \ No newline at end of file diff --git a/data/e1_h3/state-spaces--mamba-2.8b_seed7.json b/data/e1_h3/state-spaces--mamba-2.8b_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..0dafe9d1d0055a162eb3df685c001fcf6db15907 --- /dev/null +++ b/data/e1_h3/state-spaces--mamba-2.8b_seed7.json @@ -0,0 +1,350 @@ +{ + "model_type": "mamba_ssm", + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 25, + "dist_short": 10, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -6.946, + "std": 3.4195 + }, + "baseline_logp_d1000": { + "mean": -11.8226, + "std": 2.9865 + }, + "logp_gap": 4.8765, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -11.8226, + "logp_transplant_std": 2.9865 + }, + "1": { + "recovery_pct": -0.1, + "logp_transplant_mean": -11.8275, + "logp_transplant_std": 2.9928 + }, + "2": { + "recovery_pct": 0.37, + "logp_transplant_mean": -11.8047, + "logp_transplant_std": 3.1193 + }, + "3": { + "recovery_pct": 0.48, + "logp_transplant_mean": -11.799, + "logp_transplant_std": 3.0748 + }, + "4": { + "recovery_pct": 0.64, + "logp_transplant_mean": -11.7912, + "logp_transplant_std": 3.0539 + }, + "5": { + "recovery_pct": 0.26, + "logp_transplant_mean": -11.8099, + "logp_transplant_std": 3.0662 + }, + "6": { + "recovery_pct": 0.77, + "logp_transplant_mean": -11.7848, + "logp_transplant_std": 3.1346 + }, + "7": { + "recovery_pct": 0.79, + "logp_transplant_mean": -11.7839, + "logp_transplant_std": 3.1639 + }, + "8": { + "recovery_pct": 0.55, + "logp_transplant_mean": -11.7959, + "logp_transplant_std": 3.1798 + }, + "9": { + "recovery_pct": 0.14, + "logp_transplant_mean": -11.8155, + "logp_transplant_std": 3.1905 + }, + "10": { + "recovery_pct": 0.03, + "logp_transplant_mean": -11.8213, + "logp_transplant_std": 3.1787 + }, + "11": { + "recovery_pct": 0.28, + "logp_transplant_mean": -11.8089, + "logp_transplant_std": 3.2391 + }, + "12": { + "recovery_pct": 0.1, + "logp_transplant_mean": -11.8177, + "logp_transplant_std": 3.1534 + }, + "13": { + "recovery_pct": -0.14, + "logp_transplant_mean": -11.8294, + "logp_transplant_std": 3.1626 + }, + "14": { + "recovery_pct": -0.2, + "logp_transplant_mean": -11.8322, + "logp_transplant_std": 3.1637 + }, + "15": { + "recovery_pct": 0.04, + "logp_transplant_mean": -11.8206, + "logp_transplant_std": 3.2057 + }, + "16": { + "recovery_pct": 0.26, + "logp_transplant_mean": -11.8098, + "logp_transplant_std": 3.217 + }, + "17": { + "recovery_pct": 0.01, + "logp_transplant_mean": -11.8222, + "logp_transplant_std": 3.2162 + }, + "18": { + "recovery_pct": 0.16, + "logp_transplant_mean": -11.8147, + "logp_transplant_std": 3.151 + }, + "19": { + "recovery_pct": 1.22, + "logp_transplant_mean": -11.7631, + "logp_transplant_std": 3.2305 + }, + "20": { + "recovery_pct": 1.12, + "logp_transplant_mean": -11.7679, + "logp_transplant_std": 3.2154 + }, + "21": { + "recovery_pct": 1.08, + "logp_transplant_mean": -11.7699, + "logp_transplant_std": 3.2113 + }, + "22": { + "recovery_pct": 0.67, + "logp_transplant_mean": -11.7898, + "logp_transplant_std": 3.2088 + }, + "23": { + "recovery_pct": 1.18, + "logp_transplant_mean": -11.7651, + "logp_transplant_std": 3.2582 + }, + "24": { + "recovery_pct": 0.99, + "logp_transplant_mean": -11.7742, + "logp_transplant_std": 3.2714 + }, + "25": { + "recovery_pct": 0.71, + "logp_transplant_mean": -11.7881, + "logp_transplant_std": 3.2825 + }, + "26": { + "recovery_pct": 0.82, + "logp_transplant_mean": -11.7824, + "logp_transplant_std": 3.3644 + }, + "27": { + "recovery_pct": 0.93, + "logp_transplant_mean": -11.7772, + "logp_transplant_std": 3.3586 + }, + "28": { + "recovery_pct": 0.77, + "logp_transplant_mean": -11.7851, + "logp_transplant_std": 3.451 + }, + "29": { + "recovery_pct": 2.33, + "logp_transplant_mean": -11.7091, + "logp_transplant_std": 3.3612 + }, + "30": { + "recovery_pct": -1.05, + "logp_transplant_mean": -11.8736, + "logp_transplant_std": 3.2181 + }, + "31": { + "recovery_pct": 21.97, + "logp_transplant_mean": -10.751, + "logp_transplant_std": 3.0641 + }, + "32": { + "recovery_pct": 19.75, + "logp_transplant_mean": -10.8595, + "logp_transplant_std": 3.0857 + }, + "33": { + "recovery_pct": 20.01, + "logp_transplant_mean": -10.8466, + "logp_transplant_std": 3.0286 + }, + "34": { + "recovery_pct": 27.44, + "logp_transplant_mean": -10.4844, + "logp_transplant_std": 2.6999 + }, + "35": { + "recovery_pct": 28.0, + "logp_transplant_mean": -10.4571, + "logp_transplant_std": 2.6612 + }, + "36": { + "recovery_pct": 29.8, + "logp_transplant_mean": -10.3692, + "logp_transplant_std": 2.693 + }, + "37": { + "recovery_pct": 30.63, + "logp_transplant_mean": -10.329, + "logp_transplant_std": 2.7026 + }, + "38": { + "recovery_pct": 31.24, + "logp_transplant_mean": -10.2992, + "logp_transplant_std": 2.6515 + }, + "39": { + "recovery_pct": 31.86, + "logp_transplant_mean": -10.2687, + "logp_transplant_std": 2.5616 + }, + "40": { + "recovery_pct": 34.3, + "logp_transplant_mean": -10.1501, + "logp_transplant_std": 2.4601 + }, + "41": { + "recovery_pct": 33.93, + "logp_transplant_mean": -10.1681, + "logp_transplant_std": 2.497 + }, + "42": { + "recovery_pct": 73.91, + "logp_transplant_mean": -8.2182, + "logp_transplant_std": 3.4913 + }, + "43": { + "recovery_pct": 75.3, + "logp_transplant_mean": -8.1506, + "logp_transplant_std": 3.5641 + }, + "44": { + "recovery_pct": 76.57, + "logp_transplant_mean": -8.0885, + "logp_transplant_std": 3.5968 + }, + "45": { + "recovery_pct": 106.31, + "logp_transplant_mean": -6.6381, + "logp_transplant_std": 3.475 + }, + "46": { + "recovery_pct": 105.34, + "logp_transplant_mean": -6.6857, + "logp_transplant_std": 3.4952 + }, + "47": { + "recovery_pct": 104.52, + "logp_transplant_mean": -6.7256, + "logp_transplant_std": 3.5292 + }, + "48": { + "recovery_pct": 104.0, + "logp_transplant_mean": -6.7509, + "logp_transplant_std": 3.5304 + }, + "49": { + "recovery_pct": 103.56, + "logp_transplant_mean": -6.7726, + "logp_transplant_std": 3.53 + }, + "50": { + "recovery_pct": 103.53, + "logp_transplant_mean": -6.7737, + "logp_transplant_std": 3.5238 + }, + "51": { + "recovery_pct": 103.62, + "logp_transplant_mean": -6.7695, + "logp_transplant_std": 3.5199 + }, + "52": { + "recovery_pct": 103.19, + "logp_transplant_mean": -6.7906, + "logp_transplant_std": 3.5125 + }, + "53": { + "recovery_pct": 103.89, + "logp_transplant_mean": -6.7562, + "logp_transplant_std": 3.511 + }, + "54": { + "recovery_pct": 103.04, + "logp_transplant_mean": -6.7979, + "logp_transplant_std": 3.5106 + }, + "55": { + "recovery_pct": 102.12, + "logp_transplant_mean": -6.8425, + "logp_transplant_std": 3.4931 + }, + "56": { + "recovery_pct": 102.19, + "logp_transplant_mean": -6.839, + "logp_transplant_std": 3.4874 + }, + "57": { + "recovery_pct": 102.32, + "logp_transplant_mean": -6.8328, + "logp_transplant_std": 3.4831 + }, + "58": { + "recovery_pct": 102.18, + "logp_transplant_mean": -6.8399, + "logp_transplant_std": 3.4533 + }, + "59": { + "recovery_pct": 102.06, + "logp_transplant_mean": -6.8456, + "logp_transplant_std": 3.4526 + }, + "60": { + "recovery_pct": 101.65, + "logp_transplant_mean": -6.8653, + "logp_transplant_std": 3.4654 + }, + "61": { + "recovery_pct": 101.48, + "logp_transplant_mean": -6.8737, + "logp_transplant_std": 3.4512 + }, + "62": { + "recovery_pct": 101.09, + "logp_transplant_mean": -6.8927, + "logp_transplant_std": 3.4147 + }, + "63": { + "recovery_pct": 101.34, + "logp_transplant_mean": -6.8807, + "logp_transplant_std": 3.3985 + } + }, + "L_crit_90": 45, + "L_crit_99": 45, + "alpha_90": 0.7031, + "alpha_99": 0.7031, + "n_layers": 64, + "d_model": 2560, + "seed": 7, + "runtime_seconds": 7521.5 +} \ No newline at end of file diff --git a/data/e1_h3_ds100/EleutherAI--pythia-160m_seed123.json b/data/e1_h3_ds100/EleutherAI--pythia-160m_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..b1ee8b8fb0b126bf8405d3a5159a1b84dec12223 --- /dev/null +++ b/data/e1_h3_ds100/EleutherAI--pythia-160m_seed123.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 100, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -18.1032, + "std": 5.2574 + }, + "baseline_logp_d1000": { + "mean": -17.857, + "std": 5.8249 + }, + "logp_gap": -0.2462, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -17.857, + "logp_transplant_std": 5.8249 + }, + "1": { + "recovery_pct": -80.52, + "logp_transplant_mean": -17.6587, + "logp_transplant_std": 4.3183 + }, + "2": { + "recovery_pct": 69.96, + "logp_transplant_mean": -18.0292, + "logp_transplant_std": 4.2141 + }, + "3": { + "recovery_pct": 103.1, + "logp_transplant_mean": -18.1108, + "logp_transplant_std": 4.6476 + }, + "4": { + "recovery_pct": 228.21, + "logp_transplant_mean": -18.4188, + "logp_transplant_std": 4.8352 + }, + "5": { + "recovery_pct": 61.22, + "logp_transplant_mean": -18.0077, + "logp_transplant_std": 4.5548 + }, + "6": { + "recovery_pct": 235.9, + "logp_transplant_mean": -18.4378, + "logp_transplant_std": 4.8207 + }, + "7": { + "recovery_pct": 276.24, + "logp_transplant_mean": -18.5371, + "logp_transplant_std": 4.8263 + }, + "8": { + "recovery_pct": 165.0, + "logp_transplant_mean": -18.2632, + "logp_transplant_std": 4.9203 + }, + "9": { + "recovery_pct": 102.72, + "logp_transplant_mean": -18.1099, + "logp_transplant_std": 5.276 + }, + "10": { + "recovery_pct": 133.59, + "logp_transplant_mean": -18.1859, + "logp_transplant_std": 5.343 + }, + "11": { + "recovery_pct": 121.11, + "logp_transplant_mean": -18.1551, + "logp_transplant_std": 5.2159 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 12, + "d_model": 768, + "model": "EleutherAI/pythia-160m", + "seed": 123, + "runtime_seconds": 33.0 +} \ No newline at end of file diff --git a/data/e1_h3_ds100/EleutherAI--pythia-160m_seed42.json b/data/e1_h3_ds100/EleutherAI--pythia-160m_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..9f3b0d6040a30f7dfee17452ee30ab72f9ce647e --- /dev/null +++ b/data/e1_h3_ds100/EleutherAI--pythia-160m_seed42.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 100, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -23.9234, + "std": 33.5877 + }, + "baseline_logp_d1000": { + "mean": -22.5344, + "std": 32.3306 + }, + "logp_gap": -1.3891, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -22.5344, + "logp_transplant_std": 32.3306 + }, + "1": { + "recovery_pct": 28.49, + "logp_transplant_mean": -22.9301, + "logp_transplant_std": 32.4804 + }, + "2": { + "recovery_pct": 35.36, + "logp_transplant_mean": -23.0255, + "logp_transplant_std": 32.8544 + }, + "3": { + "recovery_pct": 46.46, + "logp_transplant_mean": -23.1797, + "logp_transplant_std": 32.3815 + }, + "4": { + "recovery_pct": 8.57, + "logp_transplant_mean": -22.6534, + "logp_transplant_std": 29.3835 + }, + "5": { + "recovery_pct": -38.23, + "logp_transplant_mean": -22.0033, + "logp_transplant_std": 30.0889 + }, + "6": { + "recovery_pct": -13.69, + "logp_transplant_mean": -22.3442, + "logp_transplant_std": 31.293 + }, + "7": { + "recovery_pct": 35.32, + "logp_transplant_mean": -23.025, + "logp_transplant_std": 32.1014 + }, + "8": { + "recovery_pct": 39.94, + "logp_transplant_mean": -23.0891, + "logp_transplant_std": 32.6465 + }, + "9": { + "recovery_pct": 62.99, + "logp_transplant_mean": -23.4093, + "logp_transplant_std": 33.9175 + }, + "10": { + "recovery_pct": 68.25, + "logp_transplant_mean": -23.4824, + "logp_transplant_std": 33.3208 + }, + "11": { + "recovery_pct": 97.93, + "logp_transplant_mean": -23.8947, + "logp_transplant_std": 33.6061 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 12, + "d_model": 768, + "model": "EleutherAI/pythia-160m", + "seed": 42, + "runtime_seconds": 32.8 +} \ No newline at end of file diff --git a/data/e1_h3_ds100/EleutherAI--pythia-160m_seed7.json b/data/e1_h3_ds100/EleutherAI--pythia-160m_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..abf6ad15e50df89eba8226f6553c0181ae37f9dc --- /dev/null +++ b/data/e1_h3_ds100/EleutherAI--pythia-160m_seed7.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 100, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -61.7402, + "std": 136.9717 + }, + "baseline_logp_d1000": { + "mean": -59.9124, + "std": 135.2336 + }, + "logp_gap": -1.8278, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -59.9124, + "logp_transplant_std": 135.2336 + }, + "1": { + "recovery_pct": 18.81, + "logp_transplant_mean": -60.2563, + "logp_transplant_std": 135.5896 + }, + "2": { + "recovery_pct": 56.47, + "logp_transplant_mean": -60.9445, + "logp_transplant_std": 136.1879 + }, + "3": { + "recovery_pct": 66.2, + "logp_transplant_mean": -61.1225, + "logp_transplant_std": 137.4103 + }, + "4": { + "recovery_pct": 78.64, + "logp_transplant_mean": -61.3499, + "logp_transplant_std": 136.8542 + }, + "5": { + "recovery_pct": 50.11, + "logp_transplant_mean": -60.8284, + "logp_transplant_std": 137.2768 + }, + "6": { + "recovery_pct": 80.84, + "logp_transplant_mean": -61.3901, + "logp_transplant_std": 136.7583 + }, + "7": { + "recovery_pct": 92.06, + "logp_transplant_mean": -61.5951, + "logp_transplant_std": 136.5835 + }, + "8": { + "recovery_pct": 89.05, + "logp_transplant_mean": -61.5401, + "logp_transplant_std": 136.8761 + }, + "9": { + "recovery_pct": 90.91, + "logp_transplant_mean": -61.5741, + "logp_transplant_std": 136.7656 + }, + "10": { + "recovery_pct": 89.45, + "logp_transplant_mean": -61.5474, + "logp_transplant_std": 136.6527 + }, + "11": { + "recovery_pct": 94.19, + "logp_transplant_mean": -61.6341, + "logp_transplant_std": 136.7961 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 12, + "d_model": 768, + "model": "EleutherAI/pythia-160m", + "seed": 7, + "runtime_seconds": 32.8 +} \ No newline at end of file diff --git a/data/e1_h3_ds100/EleutherAI--pythia-410m_seed123.json b/data/e1_h3_ds100/EleutherAI--pythia-410m_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..0e943b3d441239144bb66aa3415901680794f3d6 --- /dev/null +++ b/data/e1_h3_ds100/EleutherAI--pythia-410m_seed123.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 100, + "dist_short": 100, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -15.6054, + "std": 2.796 + }, + "baseline_logp_d1000": { + "mean": -14.657, + "std": 2.4179 + }, + "logp_gap": -0.9485, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.657, + "logp_transplant_std": 2.4179 + }, + "1": { + "recovery_pct": -11.59, + "logp_transplant_mean": -14.547, + "logp_transplant_std": 2.552 + }, + "2": { + "recovery_pct": 0.76, + "logp_transplant_mean": -14.6642, + "logp_transplant_std": 2.64 + }, + "3": { + "recovery_pct": 36.32, + "logp_transplant_mean": -15.0014, + "logp_transplant_std": 2.6662 + }, + "4": { + "recovery_pct": 37.99, + "logp_transplant_mean": -15.0173, + "logp_transplant_std": 2.6799 + }, + "5": { + "recovery_pct": 38.16, + "logp_transplant_mean": -15.0189, + "logp_transplant_std": 2.7044 + }, + "6": { + "recovery_pct": 45.19, + "logp_transplant_mean": -15.0856, + "logp_transplant_std": 2.7365 + }, + "7": { + "recovery_pct": 33.38, + "logp_transplant_mean": -14.9735, + "logp_transplant_std": 2.7159 + }, + "8": { + "recovery_pct": 54.78, + "logp_transplant_mean": -15.1766, + "logp_transplant_std": 2.8782 + }, + "9": { + "recovery_pct": 41.77, + "logp_transplant_mean": -15.0531, + "logp_transplant_std": 2.7814 + }, + "10": { + "recovery_pct": 34.07, + "logp_transplant_mean": -14.9801, + "logp_transplant_std": 2.9303 + }, + "11": { + "recovery_pct": 28.04, + "logp_transplant_mean": -14.9229, + "logp_transplant_std": 2.8497 + }, + "12": { + "recovery_pct": 90.29, + "logp_transplant_mean": -15.5134, + "logp_transplant_std": 2.9197 + }, + "13": { + "recovery_pct": 75.91, + "logp_transplant_mean": -15.377, + "logp_transplant_std": 2.9166 + }, + "14": { + "recovery_pct": 85.75, + "logp_transplant_mean": -15.4703, + "logp_transplant_std": 2.9955 + }, + "15": { + "recovery_pct": 83.09, + "logp_transplant_mean": -15.4451, + "logp_transplant_std": 2.935 + }, + "16": { + "recovery_pct": 88.35, + "logp_transplant_mean": -15.495, + "logp_transplant_std": 2.7281 + }, + "17": { + "recovery_pct": 101.49, + "logp_transplant_mean": -15.6195, + "logp_transplant_std": 2.5562 + }, + "18": { + "recovery_pct": 112.69, + "logp_transplant_mean": -15.7258, + "logp_transplant_std": 2.5939 + }, + "19": { + "recovery_pct": 104.18, + "logp_transplant_mean": -15.6451, + "logp_transplant_std": 2.6355 + }, + "20": { + "recovery_pct": 114.0, + "logp_transplant_mean": -15.7382, + "logp_transplant_std": 2.6871 + }, + "21": { + "recovery_pct": 107.51, + "logp_transplant_mean": -15.6767, + "logp_transplant_std": 2.7956 + }, + "22": { + "recovery_pct": 96.09, + "logp_transplant_mean": -15.5684, + "logp_transplant_std": 2.7521 + }, + "23": { + "recovery_pct": 97.62, + "logp_transplant_mean": -15.5828, + "logp_transplant_std": 2.7637 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 24, + "d_model": 1024, + "model": "EleutherAI/pythia-410m", + "seed": 123, + "runtime_seconds": 331.9 +} \ No newline at end of file diff --git a/data/e1_h3_ds100/EleutherAI--pythia-410m_seed42.json b/data/e1_h3_ds100/EleutherAI--pythia-410m_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..2c058f47e745565f64f56c2b835cc3cd8418056d --- /dev/null +++ b/data/e1_h3_ds100/EleutherAI--pythia-410m_seed42.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 100, + "dist_short": 100, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -15.3018, + "std": 2.884 + }, + "baseline_logp_d1000": { + "mean": -14.6291, + "std": 2.388 + }, + "logp_gap": -0.6727, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.6291, + "logp_transplant_std": 2.388 + }, + "1": { + "recovery_pct": -57.53, + "logp_transplant_mean": -14.2421, + "logp_transplant_std": 2.5542 + }, + "2": { + "recovery_pct": -22.42, + "logp_transplant_mean": -14.4783, + "logp_transplant_std": 2.5263 + }, + "3": { + "recovery_pct": 32.91, + "logp_transplant_mean": -14.8504, + "logp_transplant_std": 2.3763 + }, + "4": { + "recovery_pct": 33.57, + "logp_transplant_mean": -14.8549, + "logp_transplant_std": 2.3955 + }, + "5": { + "recovery_pct": 27.95, + "logp_transplant_mean": -14.8171, + "logp_transplant_std": 2.4338 + }, + "6": { + "recovery_pct": 9.5, + "logp_transplant_mean": -14.693, + "logp_transplant_std": 2.4885 + }, + "7": { + "recovery_pct": 34.76, + "logp_transplant_mean": -14.8629, + "logp_transplant_std": 2.445 + }, + "8": { + "recovery_pct": 22.13, + "logp_transplant_mean": -14.7779, + "logp_transplant_std": 2.4014 + }, + "9": { + "recovery_pct": 14.4, + "logp_transplant_mean": -14.7259, + "logp_transplant_std": 2.5819 + }, + "10": { + "recovery_pct": -52.92, + "logp_transplant_mean": -14.2731, + "logp_transplant_std": 2.6676 + }, + "11": { + "recovery_pct": -3.28, + "logp_transplant_mean": -14.607, + "logp_transplant_std": 2.8051 + }, + "12": { + "recovery_pct": 26.02, + "logp_transplant_mean": -14.8041, + "logp_transplant_std": 2.7013 + }, + "13": { + "recovery_pct": 18.83, + "logp_transplant_mean": -14.7557, + "logp_transplant_std": 2.7115 + }, + "14": { + "recovery_pct": 11.37, + "logp_transplant_mean": -14.7056, + "logp_transplant_std": 2.7423 + }, + "15": { + "recovery_pct": 10.8, + "logp_transplant_mean": -14.7017, + "logp_transplant_std": 2.7057 + }, + "16": { + "recovery_pct": 62.77, + "logp_transplant_mean": -15.0513, + "logp_transplant_std": 2.8131 + }, + "17": { + "recovery_pct": 70.83, + "logp_transplant_mean": -15.1055, + "logp_transplant_std": 3.0594 + }, + "18": { + "recovery_pct": 72.19, + "logp_transplant_mean": -15.1147, + "logp_transplant_std": 2.9744 + }, + "19": { + "recovery_pct": 61.11, + "logp_transplant_mean": -15.0401, + "logp_transplant_std": 2.7854 + }, + "20": { + "recovery_pct": 66.54, + "logp_transplant_mean": -15.0767, + "logp_transplant_std": 2.8673 + }, + "21": { + "recovery_pct": 87.93, + "logp_transplant_mean": -15.2205, + "logp_transplant_std": 2.8033 + }, + "22": { + "recovery_pct": 85.88, + "logp_transplant_mean": -15.2068, + "logp_transplant_std": 2.8433 + }, + "23": { + "recovery_pct": 89.73, + "logp_transplant_mean": -15.2327, + "logp_transplant_std": 2.8731 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 24, + "d_model": 1024, + "model": "EleutherAI/pythia-410m", + "seed": 42, + "runtime_seconds": 329.2 +} \ No newline at end of file diff --git a/data/e1_h3_ds100/EleutherAI--pythia-410m_seed7.json b/data/e1_h3_ds100/EleutherAI--pythia-410m_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..b252a3a1f2df7346655bb5ed628ab066b7f4579e --- /dev/null +++ b/data/e1_h3_ds100/EleutherAI--pythia-410m_seed7.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 100, + "dist_short": 100, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -15.287, + "std": 2.6746 + }, + "baseline_logp_d1000": { + "mean": -14.9379, + "std": 2.6239 + }, + "logp_gap": -0.3491, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.9379, + "logp_transplant_std": 2.6239 + }, + "1": { + "recovery_pct": 16.21, + "logp_transplant_mean": -14.9945, + "logp_transplant_std": 2.6022 + }, + "2": { + "recovery_pct": 30.75, + "logp_transplant_mean": -15.0453, + "logp_transplant_std": 2.6162 + }, + "3": { + "recovery_pct": 152.12, + "logp_transplant_mean": -15.4689, + "logp_transplant_std": 2.5791 + }, + "4": { + "recovery_pct": 130.79, + "logp_transplant_mean": -15.3945, + "logp_transplant_std": 2.5325 + }, + "5": { + "recovery_pct": 136.04, + "logp_transplant_mean": -15.4128, + "logp_transplant_std": 2.5614 + }, + "6": { + "recovery_pct": 143.77, + "logp_transplant_mean": -15.4398, + "logp_transplant_std": 2.5954 + }, + "7": { + "recovery_pct": 125.08, + "logp_transplant_mean": -15.3745, + "logp_transplant_std": 2.656 + }, + "8": { + "recovery_pct": 148.93, + "logp_transplant_mean": -15.4578, + "logp_transplant_std": 2.702 + }, + "9": { + "recovery_pct": 105.39, + "logp_transplant_mean": -15.3058, + "logp_transplant_std": 2.5581 + }, + "10": { + "recovery_pct": 61.92, + "logp_transplant_mean": -15.1541, + "logp_transplant_std": 2.5592 + }, + "11": { + "recovery_pct": 100.96, + "logp_transplant_mean": -15.2903, + "logp_transplant_std": 2.4941 + }, + "12": { + "recovery_pct": 82.22, + "logp_transplant_mean": -15.2249, + "logp_transplant_std": 2.5403 + }, + "13": { + "recovery_pct": 142.91, + "logp_transplant_mean": -15.4368, + "logp_transplant_std": 2.692 + }, + "14": { + "recovery_pct": 170.52, + "logp_transplant_mean": -15.5331, + "logp_transplant_std": 2.6522 + }, + "15": { + "recovery_pct": 165.42, + "logp_transplant_mean": -15.5153, + "logp_transplant_std": 2.5123 + }, + "16": { + "recovery_pct": 211.44, + "logp_transplant_mean": -15.676, + "logp_transplant_std": 2.8152 + }, + "17": { + "recovery_pct": 176.86, + "logp_transplant_mean": -15.5553, + "logp_transplant_std": 2.6863 + }, + "18": { + "recovery_pct": 147.93, + "logp_transplant_mean": -15.4543, + "logp_transplant_std": 2.7503 + }, + "19": { + "recovery_pct": 87.01, + "logp_transplant_mean": -15.2416, + "logp_transplant_std": 2.5612 + }, + "20": { + "recovery_pct": 84.12, + "logp_transplant_mean": -15.2315, + "logp_transplant_std": 2.5532 + }, + "21": { + "recovery_pct": 117.43, + "logp_transplant_mean": -15.3478, + "logp_transplant_std": 2.7143 + }, + "22": { + "recovery_pct": 106.91, + "logp_transplant_mean": -15.3111, + "logp_transplant_std": 2.7424 + }, + "23": { + "recovery_pct": 91.55, + "logp_transplant_mean": -15.2575, + "logp_transplant_std": 2.6171 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 24, + "d_model": 1024, + "model": "EleutherAI/pythia-410m", + "seed": 7, + "runtime_seconds": 330.0 +} \ No newline at end of file diff --git a/data/e1_h3_ds50/EleutherAI--pythia-160m_seed123.json b/data/e1_h3_ds50/EleutherAI--pythia-160m_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..3d652d7fb9ba4247248832bcd8a10a70e4c8a5dc --- /dev/null +++ b/data/e1_h3_ds50/EleutherAI--pythia-160m_seed123.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 50, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -19.8087, + "std": 7.7447 + }, + "baseline_logp_d1000": { + "mean": -17.857, + "std": 5.8249 + }, + "logp_gap": -1.9517, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -17.857, + "logp_transplant_std": 5.8249 + }, + "1": { + "recovery_pct": 9.76, + "logp_transplant_mean": -18.0475, + "logp_transplant_std": 4.4382 + }, + "2": { + "recovery_pct": 5.5, + "logp_transplant_mean": -17.9643, + "logp_transplant_std": 4.6021 + }, + "3": { + "recovery_pct": -8.87, + "logp_transplant_mean": -17.6839, + "logp_transplant_std": 3.7088 + }, + "4": { + "recovery_pct": 12.57, + "logp_transplant_mean": -18.1023, + "logp_transplant_std": 5.1753 + }, + "5": { + "recovery_pct": 42.93, + "logp_transplant_mean": -18.6948, + "logp_transplant_std": 6.4078 + }, + "6": { + "recovery_pct": 72.81, + "logp_transplant_mean": -19.278, + "logp_transplant_std": 6.0575 + }, + "7": { + "recovery_pct": 95.79, + "logp_transplant_mean": -19.7265, + "logp_transplant_std": 6.118 + }, + "8": { + "recovery_pct": 72.65, + "logp_transplant_mean": -19.2749, + "logp_transplant_std": 5.3456 + }, + "9": { + "recovery_pct": 90.98, + "logp_transplant_mean": -19.6326, + "logp_transplant_std": 6.8692 + }, + "10": { + "recovery_pct": 78.9, + "logp_transplant_mean": -19.3969, + "logp_transplant_std": 7.1468 + }, + "11": { + "recovery_pct": 98.53, + "logp_transplant_mean": -19.7799, + "logp_transplant_std": 7.7156 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 12, + "d_model": 768, + "model": "EleutherAI/pythia-160m", + "seed": 123, + "runtime_seconds": 32.2 +} \ No newline at end of file diff --git a/data/e1_h3_ds50/EleutherAI--pythia-160m_seed42.json b/data/e1_h3_ds50/EleutherAI--pythia-160m_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..487820a8a928f7de67c2fcefa3b10b9ab5d522f7 --- /dev/null +++ b/data/e1_h3_ds50/EleutherAI--pythia-160m_seed42.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 50, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -24.3881, + "std": 31.7154 + }, + "baseline_logp_d1000": { + "mean": -22.5344, + "std": 32.3306 + }, + "logp_gap": -1.8537, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -22.5344, + "logp_transplant_std": 32.3306 + }, + "1": { + "recovery_pct": 22.99, + "logp_transplant_mean": -22.9606, + "logp_transplant_std": 33.0628 + }, + "2": { + "recovery_pct": 10.23, + "logp_transplant_mean": -22.724, + "logp_transplant_std": 32.622 + }, + "3": { + "recovery_pct": 40.82, + "logp_transplant_mean": -23.291, + "logp_transplant_std": 32.9937 + }, + "4": { + "recovery_pct": 3.37, + "logp_transplant_mean": -22.5968, + "logp_transplant_std": 31.8858 + }, + "5": { + "recovery_pct": 44.34, + "logp_transplant_mean": -23.3564, + "logp_transplant_std": 32.2168 + }, + "6": { + "recovery_pct": 76.15, + "logp_transplant_mean": -23.9459, + "logp_transplant_std": 32.6392 + }, + "7": { + "recovery_pct": 82.6, + "logp_transplant_mean": -24.0656, + "logp_transplant_std": 32.1924 + }, + "8": { + "recovery_pct": 78.41, + "logp_transplant_mean": -23.9878, + "logp_transplant_std": 32.6471 + }, + "9": { + "recovery_pct": 108.37, + "logp_transplant_mean": -24.5433, + "logp_transplant_std": 32.5635 + }, + "10": { + "recovery_pct": 84.36, + "logp_transplant_mean": -24.0983, + "logp_transplant_std": 31.7546 + }, + "11": { + "recovery_pct": 94.5, + "logp_transplant_mean": -24.2862, + "logp_transplant_std": 31.727 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 12, + "d_model": 768, + "model": "EleutherAI/pythia-160m", + "seed": 42, + "runtime_seconds": 32.5 +} \ No newline at end of file diff --git a/data/e1_h3_ds50/EleutherAI--pythia-160m_seed7.json b/data/e1_h3_ds50/EleutherAI--pythia-160m_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..7f50121ed91a5ec5a955857f34b849a2df8125eb --- /dev/null +++ b/data/e1_h3_ds50/EleutherAI--pythia-160m_seed7.json @@ -0,0 +1,91 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 50, + "dist_short": 50, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -61.9463, + "std": 134.5358 + }, + "baseline_logp_d1000": { + "mean": -59.9124, + "std": 135.2336 + }, + "logp_gap": -2.0338, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -59.9124, + "logp_transplant_std": 135.2336 + }, + "1": { + "recovery_pct": 55.8, + "logp_transplant_mean": -61.0473, + "logp_transplant_std": 135.2225 + }, + "2": { + "recovery_pct": 52.73, + "logp_transplant_mean": -60.9849, + "logp_transplant_std": 135.3461 + }, + "3": { + "recovery_pct": 49.56, + "logp_transplant_mean": -60.9204, + "logp_transplant_std": 136.2581 + }, + "4": { + "recovery_pct": 36.01, + "logp_transplant_mean": -60.6447, + "logp_transplant_std": 136.3713 + }, + "5": { + "recovery_pct": 38.58, + "logp_transplant_mean": -60.697, + "logp_transplant_std": 136.1015 + }, + "6": { + "recovery_pct": 90.67, + "logp_transplant_mean": -61.7565, + "logp_transplant_std": 135.1875 + }, + "7": { + "recovery_pct": 90.11, + "logp_transplant_mean": -61.7452, + "logp_transplant_std": 134.1286 + }, + "8": { + "recovery_pct": 87.86, + "logp_transplant_mean": -61.6993, + "logp_transplant_std": 134.3207 + }, + "9": { + "recovery_pct": 92.34, + "logp_transplant_mean": -61.7906, + "logp_transplant_std": 133.9563 + }, + "10": { + "recovery_pct": 93.73, + "logp_transplant_mean": -61.8188, + "logp_transplant_std": 134.8652 + }, + "11": { + "recovery_pct": 98.15, + "logp_transplant_mean": -61.9086, + "logp_transplant_std": 134.6089 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 12, + "d_model": 768, + "model": "EleutherAI/pythia-160m", + "seed": 7, + "runtime_seconds": 32.4 +} \ No newline at end of file diff --git a/data/e1_h3_ds50/EleutherAI--pythia-410m_seed123.json b/data/e1_h3_ds50/EleutherAI--pythia-410m_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..db4ccf22ac706682c5da9e843d7b957cfc12854b --- /dev/null +++ b/data/e1_h3_ds50/EleutherAI--pythia-410m_seed123.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 100, + "dist_short": 50, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -15.6745, + "std": 3.1729 + }, + "baseline_logp_d1000": { + "mean": -14.657, + "std": 2.4179 + }, + "logp_gap": -1.0176, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.657, + "logp_transplant_std": 2.4179 + }, + "1": { + "recovery_pct": -2.28, + "logp_transplant_mean": -14.6337, + "logp_transplant_std": 2.6353 + }, + "2": { + "recovery_pct": 11.8, + "logp_transplant_mean": -14.7771, + "logp_transplant_std": 2.6941 + }, + "3": { + "recovery_pct": 41.48, + "logp_transplant_mean": -15.079, + "logp_transplant_std": 2.7358 + }, + "4": { + "recovery_pct": 41.19, + "logp_transplant_mean": -15.0761, + "logp_transplant_std": 2.7304 + }, + "5": { + "recovery_pct": 43.63, + "logp_transplant_mean": -15.101, + "logp_transplant_std": 2.8256 + }, + "6": { + "recovery_pct": 36.99, + "logp_transplant_mean": -15.0334, + "logp_transplant_std": 2.7182 + }, + "7": { + "recovery_pct": 36.38, + "logp_transplant_mean": -15.0272, + "logp_transplant_std": 2.7024 + }, + "8": { + "recovery_pct": 51.08, + "logp_transplant_mean": -15.1767, + "logp_transplant_std": 2.8199 + }, + "9": { + "recovery_pct": 38.62, + "logp_transplant_mean": -15.0499, + "logp_transplant_std": 2.8816 + }, + "10": { + "recovery_pct": 62.1, + "logp_transplant_mean": -15.2889, + "logp_transplant_std": 2.8358 + }, + "11": { + "recovery_pct": 71.53, + "logp_transplant_mean": -15.3849, + "logp_transplant_std": 2.9841 + }, + "12": { + "recovery_pct": 85.55, + "logp_transplant_mean": -15.5275, + "logp_transplant_std": 3.0098 + }, + "13": { + "recovery_pct": 80.47, + "logp_transplant_mean": -15.4758, + "logp_transplant_std": 3.0469 + }, + "14": { + "recovery_pct": 103.18, + "logp_transplant_mean": -15.7069, + "logp_transplant_std": 3.0738 + }, + "15": { + "recovery_pct": 89.69, + "logp_transplant_mean": -15.5696, + "logp_transplant_std": 2.9538 + }, + "16": { + "recovery_pct": 92.87, + "logp_transplant_mean": -15.602, + "logp_transplant_std": 2.9948 + }, + "17": { + "recovery_pct": 124.21, + "logp_transplant_mean": -15.9209, + "logp_transplant_std": 2.782 + }, + "18": { + "recovery_pct": 112.96, + "logp_transplant_mean": -15.8064, + "logp_transplant_std": 3.0524 + }, + "19": { + "recovery_pct": 96.33, + "logp_transplant_mean": -15.6372, + "logp_transplant_std": 2.9933 + }, + "20": { + "recovery_pct": 114.9, + "logp_transplant_mean": -15.8262, + "logp_transplant_std": 3.27 + }, + "21": { + "recovery_pct": 107.54, + "logp_transplant_mean": -15.7512, + "logp_transplant_std": 3.3463 + }, + "22": { + "recovery_pct": 104.4, + "logp_transplant_mean": -15.7193, + "logp_transplant_std": 3.2789 + }, + "23": { + "recovery_pct": 97.17, + "logp_transplant_mean": -15.6458, + "logp_transplant_std": 3.2092 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 24, + "d_model": 1024, + "model": "EleutherAI/pythia-410m", + "seed": 123, + "runtime_seconds": 332.1 +} \ No newline at end of file diff --git a/data/e1_h3_ds50/EleutherAI--pythia-410m_seed42.json b/data/e1_h3_ds50/EleutherAI--pythia-410m_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..3f5e563d9a28c4818a64abf3a047d967171a7c4a --- /dev/null +++ b/data/e1_h3_ds50/EleutherAI--pythia-410m_seed42.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 100, + "dist_short": 50, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -15.2206, + "std": 2.7685 + }, + "baseline_logp_d1000": { + "mean": -14.6291, + "std": 2.388 + }, + "logp_gap": -0.5915, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.6291, + "logp_transplant_std": 2.388 + }, + "1": { + "recovery_pct": -63.71, + "logp_transplant_mean": -14.2522, + "logp_transplant_std": 2.421 + }, + "2": { + "recovery_pct": -45.17, + "logp_transplant_mean": -14.3619, + "logp_transplant_std": 2.3405 + }, + "3": { + "recovery_pct": 47.33, + "logp_transplant_mean": -14.909, + "logp_transplant_std": 2.4276 + }, + "4": { + "recovery_pct": 51.54, + "logp_transplant_mean": -14.9339, + "logp_transplant_std": 2.4745 + }, + "5": { + "recovery_pct": 45.08, + "logp_transplant_mean": -14.8957, + "logp_transplant_std": 2.4652 + }, + "6": { + "recovery_pct": 42.73, + "logp_transplant_mean": -14.8818, + "logp_transplant_std": 2.4158 + }, + "7": { + "recovery_pct": 62.32, + "logp_transplant_mean": -14.9977, + "logp_transplant_std": 2.4048 + }, + "8": { + "recovery_pct": 13.83, + "logp_transplant_mean": -14.7109, + "logp_transplant_std": 2.4849 + }, + "9": { + "recovery_pct": 8.04, + "logp_transplant_mean": -14.6766, + "logp_transplant_std": 2.6077 + }, + "10": { + "recovery_pct": -34.92, + "logp_transplant_mean": -14.4225, + "logp_transplant_std": 2.5114 + }, + "11": { + "recovery_pct": 36.56, + "logp_transplant_mean": -14.8453, + "logp_transplant_std": 2.8356 + }, + "12": { + "recovery_pct": 47.3, + "logp_transplant_mean": -14.9089, + "logp_transplant_std": 2.7559 + }, + "13": { + "recovery_pct": 53.15, + "logp_transplant_mean": -14.9434, + "logp_transplant_std": 2.7645 + }, + "14": { + "recovery_pct": 70.03, + "logp_transplant_mean": -15.0433, + "logp_transplant_std": 2.7776 + }, + "15": { + "recovery_pct": 44.23, + "logp_transplant_mean": -14.8907, + "logp_transplant_std": 2.6925 + }, + "16": { + "recovery_pct": 47.01, + "logp_transplant_mean": -14.9071, + "logp_transplant_std": 2.6983 + }, + "17": { + "recovery_pct": 87.04, + "logp_transplant_mean": -15.1439, + "logp_transplant_std": 2.7331 + }, + "18": { + "recovery_pct": 87.88, + "logp_transplant_mean": -15.1489, + "logp_transplant_std": 2.7205 + }, + "19": { + "recovery_pct": 102.37, + "logp_transplant_mean": -15.2346, + "logp_transplant_std": 2.9277 + }, + "20": { + "recovery_pct": 116.26, + "logp_transplant_mean": -15.3167, + "logp_transplant_std": 2.8439 + }, + "21": { + "recovery_pct": 125.02, + "logp_transplant_mean": -15.3685, + "logp_transplant_std": 2.9143 + }, + "22": { + "recovery_pct": 96.31, + "logp_transplant_mean": -15.1987, + "logp_transplant_std": 2.717 + }, + "23": { + "recovery_pct": 91.91, + "logp_transplant_mean": -15.1727, + "logp_transplant_std": 2.778 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 24, + "d_model": 1024, + "model": "EleutherAI/pythia-410m", + "seed": 42, + "runtime_seconds": 330.1 +} \ No newline at end of file diff --git a/data/e1_h3_ds50/EleutherAI--pythia-410m_seed7.json b/data/e1_h3_ds50/EleutherAI--pythia-410m_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..58a2d7defdf703b5bc88f5892d783da78e9bdc35 --- /dev/null +++ b/data/e1_h3_ds50/EleutherAI--pythia-410m_seed7.json @@ -0,0 +1,151 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 100, + "dist_short": 50, + "dist_long": 1000, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -15.0643, + "std": 2.8882 + }, + "baseline_logp_d1000": { + "mean": -14.9379, + "std": 2.6239 + }, + "logp_gap": -0.1264, + "layer_sweep": { + "0": { + "recovery_pct": -0.0, + "logp_transplant_mean": -14.9379, + "logp_transplant_std": 2.6239 + }, + "1": { + "recovery_pct": -147.73, + "logp_transplant_mean": -14.7512, + "logp_transplant_std": 2.6008 + }, + "2": { + "recovery_pct": -48.77, + "logp_transplant_mean": -14.8763, + "logp_transplant_std": 2.6189 + }, + "3": { + "recovery_pct": 261.74, + "logp_transplant_mean": -15.2687, + "logp_transplant_std": 2.4871 + }, + "4": { + "recovery_pct": 247.25, + "logp_transplant_mean": -15.2504, + "logp_transplant_std": 2.5574 + }, + "5": { + "recovery_pct": 291.16, + "logp_transplant_mean": -15.3059, + "logp_transplant_std": 2.4462 + }, + "6": { + "recovery_pct": 277.84, + "logp_transplant_mean": -15.2891, + "logp_transplant_std": 2.4881 + }, + "7": { + "recovery_pct": 308.07, + "logp_transplant_mean": -15.3273, + "logp_transplant_std": 2.592 + }, + "8": { + "recovery_pct": 197.76, + "logp_transplant_mean": -15.1879, + "logp_transplant_std": 2.6484 + }, + "9": { + "recovery_pct": 164.29, + "logp_transplant_mean": -15.1456, + "logp_transplant_std": 2.5619 + }, + "10": { + "recovery_pct": 32.01, + "logp_transplant_mean": -14.9784, + "logp_transplant_std": 2.5849 + }, + "11": { + "recovery_pct": 215.9, + "logp_transplant_mean": -15.2108, + "logp_transplant_std": 2.5231 + }, + "12": { + "recovery_pct": 367.77, + "logp_transplant_mean": -15.4027, + "logp_transplant_std": 2.5381 + }, + "13": { + "recovery_pct": 314.19, + "logp_transplant_mean": -15.335, + "logp_transplant_std": 2.5856 + }, + "14": { + "recovery_pct": 416.82, + "logp_transplant_mean": -15.4647, + "logp_transplant_std": 2.6024 + }, + "15": { + "recovery_pct": 512.95, + "logp_transplant_mean": -15.5862, + "logp_transplant_std": 2.7555 + }, + "16": { + "recovery_pct": 528.49, + "logp_transplant_mean": -15.6058, + "logp_transplant_std": 2.7083 + }, + "17": { + "recovery_pct": 646.78, + "logp_transplant_mean": -15.7553, + "logp_transplant_std": 2.6579 + }, + "18": { + "recovery_pct": 319.88, + "logp_transplant_mean": -15.3422, + "logp_transplant_std": 2.8567 + }, + "19": { + "recovery_pct": 177.52, + "logp_transplant_mean": -15.1623, + "logp_transplant_std": 2.9077 + }, + "20": { + "recovery_pct": 176.48, + "logp_transplant_mean": -15.161, + "logp_transplant_std": 2.8369 + }, + "21": { + "recovery_pct": 148.19, + "logp_transplant_mean": -15.1252, + "logp_transplant_std": 2.9844 + }, + "22": { + "recovery_pct": 100.59, + "logp_transplant_mean": -15.065, + "logp_transplant_std": 2.9196 + }, + "23": { + "recovery_pct": 91.03, + "logp_transplant_mean": -15.053, + "logp_transplant_std": 2.8959 + } + }, + "L_crit_90": null, + "L_crit_99": null, + "alpha_90": null, + "alpha_99": null, + "recovery_at_Lcrit": null, + "n_layers": 24, + "d_model": 1024, + "model": "EleutherAI/pythia-410m", + "seed": 7, + "runtime_seconds": 331.5 +} \ No newline at end of file diff --git a/data/e1_test/EleutherAI--pythia-1b_seed123.json b/data/e1_test/EleutherAI--pythia-1b_seed123.json new file mode 100644 index 0000000000000000000000000000000000000000..449fdd90e481c5acc4e870c034cef0a2c09074bc --- /dev/null +++ b/data/e1_test/EleutherAI--pythia-1b_seed123.json @@ -0,0 +1,111 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 2, + "dist_short": 10, + "dist_long": 2048, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -10.4631, + "std": 0.2133 + }, + "baseline_logp_d1000": { + "mean": -11.0983, + "std": 0.5716 + }, + "logp_gap": 0.6351, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -11.0983, + "logp_transplant_std": 0.5716 + }, + "1": { + "recovery_pct": -18.25, + "logp_transplant_mean": -11.2142, + "logp_transplant_std": 0.3048 + }, + "2": { + "recovery_pct": -39.25, + "logp_transplant_mean": -11.3476, + "logp_transplant_std": 0.3181 + }, + "3": { + "recovery_pct": 38.32, + "logp_transplant_mean": -10.8549, + "logp_transplant_std": 0.9846 + }, + "4": { + "recovery_pct": 39.45, + "logp_transplant_mean": -10.8477, + "logp_transplant_std": 1.001 + }, + "5": { + "recovery_pct": 20.65, + "logp_transplant_mean": -10.9671, + "logp_transplant_std": 0.8819 + }, + "6": { + "recovery_pct": 35.9, + "logp_transplant_mean": -10.8702, + "logp_transplant_std": 0.6389 + }, + "7": { + "recovery_pct": 41.22, + "logp_transplant_mean": -10.8364, + "logp_transplant_std": 0.5933 + }, + "8": { + "recovery_pct": 72.81, + "logp_transplant_mean": -10.6359, + "logp_transplant_std": 0.5532 + }, + "9": { + "recovery_pct": 68.03, + "logp_transplant_mean": -10.6662, + "logp_transplant_std": 0.2462 + }, + "10": { + "recovery_pct": 64.06, + "logp_transplant_mean": -10.6914, + "logp_transplant_std": 0.1793 + }, + "11": { + "recovery_pct": 80.26, + "logp_transplant_mean": -10.5885, + "logp_transplant_std": 0.0766 + }, + "12": { + "recovery_pct": 73.92, + "logp_transplant_mean": -10.6288, + "logp_transplant_std": 0.002 + }, + "13": { + "recovery_pct": 144.13, + "logp_transplant_mean": -10.1829, + "logp_transplant_std": 0.2098 + }, + "14": { + "recovery_pct": 76.59, + "logp_transplant_mean": -10.6118, + "logp_transplant_std": 0.2318 + }, + "15": { + "recovery_pct": 131.19, + "logp_transplant_mean": -10.2651, + "logp_transplant_std": 0.1007 + } + }, + "L_crit_90": 13, + "L_crit_99": 13, + "alpha_90": 0.8125, + "alpha_99": 0.8125, + "recovery_at_Lcrit": 1.4413, + "n_layers": 16, + "d_model": 2048, + "model": "EleutherAI/pythia-1b", + "seed": 123, + "runtime_seconds": 15.3 +} \ No newline at end of file diff --git a/data/e1_test/EleutherAI--pythia-1b_seed42.json b/data/e1_test/EleutherAI--pythia-1b_seed42.json new file mode 100644 index 0000000000000000000000000000000000000000..5ec63320cccf6150c83156e352e575783668b9ff --- /dev/null +++ b/data/e1_test/EleutherAI--pythia-1b_seed42.json @@ -0,0 +1,111 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 2, + "dist_short": 10, + "dist_long": 2048, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -10.2814, + "std": 0.2195 + }, + "baseline_logp_d1000": { + "mean": -12.5204, + "std": 0.6399 + }, + "logp_gap": 2.239, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -12.5204, + "logp_transplant_std": 0.6399 + }, + "1": { + "recovery_pct": -2.24, + "logp_transplant_mean": -12.5706, + "logp_transplant_std": 0.5277 + }, + "2": { + "recovery_pct": -5.55, + "logp_transplant_mean": -12.6447, + "logp_transplant_std": 0.4928 + }, + "3": { + "recovery_pct": -4.76, + "logp_transplant_mean": -12.6271, + "logp_transplant_std": 0.5457 + }, + "4": { + "recovery_pct": -0.28, + "logp_transplant_mean": -12.5266, + "logp_transplant_std": 0.6241 + }, + "5": { + "recovery_pct": 6.44, + "logp_transplant_mean": -12.3762, + "logp_transplant_std": 0.8001 + }, + "6": { + "recovery_pct": 9.12, + "logp_transplant_mean": -12.3162, + "logp_transplant_std": 0.8337 + }, + "7": { + "recovery_pct": 10.78, + "logp_transplant_mean": -12.2791, + "logp_transplant_std": 0.809 + }, + "8": { + "recovery_pct": 16.27, + "logp_transplant_mean": -12.1561, + "logp_transplant_std": 0.6089 + }, + "9": { + "recovery_pct": 35.59, + "logp_transplant_mean": -11.7235, + "logp_transplant_std": 0.5192 + }, + "10": { + "recovery_pct": 64.45, + "logp_transplant_mean": -11.0774, + "logp_transplant_std": 0.4215 + }, + "11": { + "recovery_pct": 64.73, + "logp_transplant_mean": -11.0711, + "logp_transplant_std": 0.4269 + }, + "12": { + "recovery_pct": 72.38, + "logp_transplant_mean": -10.8999, + "logp_transplant_std": 0.3679 + }, + "13": { + "recovery_pct": 70.9, + "logp_transplant_mean": -10.9329, + "logp_transplant_std": 0.4293 + }, + "14": { + "recovery_pct": 74.75, + "logp_transplant_mean": -10.8468, + "logp_transplant_std": 0.2333 + }, + "15": { + "recovery_pct": 94.26, + "logp_transplant_mean": -10.41, + "logp_transplant_std": 0.2274 + } + }, + "L_crit_90": 15, + "L_crit_99": null, + "alpha_90": 0.9375, + "alpha_99": null, + "recovery_at_Lcrit": 0.9426, + "n_layers": 16, + "d_model": 2048, + "model": "EleutherAI/pythia-1b", + "seed": 42, + "runtime_seconds": 15.5 +} \ No newline at end of file diff --git a/data/e1_test/EleutherAI--pythia-1b_seed7.json b/data/e1_test/EleutherAI--pythia-1b_seed7.json new file mode 100644 index 0000000000000000000000000000000000000000..f487893b307b26689cd55ae04f41584a9d7dcda5 --- /dev/null +++ b/data/e1_test/EleutherAI--pythia-1b_seed7.json @@ -0,0 +1,111 @@ +{ + "tau_90": 0.9, + "tau_99": 0.99, + "N_prompts": 2, + "dist_short": 10, + "dist_long": 2048, + "vocab_low": 1000, + "vocab_high": 49000, + "n_prefix": 20, + "ref_logp_d10": { + "mean": -9.2085, + "std": 1.2153 + }, + "baseline_logp_d1000": { + "mean": -12.8734, + "std": 0.1521 + }, + "logp_gap": 3.6649, + "layer_sweep": { + "0": { + "recovery_pct": 0.0, + "logp_transplant_mean": -12.8734, + "logp_transplant_std": 0.1521 + }, + "1": { + "recovery_pct": -1.05, + "logp_transplant_mean": -12.9117, + "logp_transplant_std": 0.078 + }, + "2": { + "recovery_pct": -3.08, + "logp_transplant_mean": -12.9862, + "logp_transplant_std": 0.0051 + }, + "3": { + "recovery_pct": -1.04, + "logp_transplant_mean": -12.9115, + "logp_transplant_std": 0.0218 + }, + "4": { + "recovery_pct": -2.51, + "logp_transplant_mean": -12.9655, + "logp_transplant_std": 0.0895 + }, + "5": { + "recovery_pct": 3.71, + "logp_transplant_mean": -12.7375, + "logp_transplant_std": 0.1918 + }, + "6": { + "recovery_pct": 14.42, + "logp_transplant_mean": -12.3449, + "logp_transplant_std": 0.1765 + }, + "7": { + "recovery_pct": 16.42, + "logp_transplant_mean": -12.2718, + "logp_transplant_std": 0.12 + }, + "8": { + "recovery_pct": 34.6, + "logp_transplant_mean": -11.6052, + "logp_transplant_std": 0.1632 + }, + "9": { + "recovery_pct": 40.13, + "logp_transplant_mean": -11.4025, + "logp_transplant_std": 0.2572 + }, + "10": { + "recovery_pct": 49.52, + "logp_transplant_mean": -11.0585, + "logp_transplant_std": 0.0246 + }, + "11": { + "recovery_pct": 54.47, + "logp_transplant_mean": -10.8771, + "logp_transplant_std": 0.0591 + }, + "12": { + "recovery_pct": 77.39, + "logp_transplant_mean": -10.0373, + "logp_transplant_std": 0.3924 + }, + "13": { + "recovery_pct": 75.74, + "logp_transplant_mean": -10.0974, + "logp_transplant_std": 0.1186 + }, + "14": { + "recovery_pct": 72.11, + "logp_transplant_mean": -10.2305, + "logp_transplant_std": 0.4037 + }, + "15": { + "recovery_pct": 90.08, + "logp_transplant_mean": -9.5719, + "logp_transplant_std": 0.7931 + } + }, + "L_crit_90": 15, + "L_crit_99": null, + "alpha_90": 0.9375, + "alpha_99": null, + "recovery_at_Lcrit": 0.9008, + "n_layers": 16, + "d_model": 2048, + "model": "EleutherAI/pythia-1b", + "seed": 7, + "runtime_seconds": 15.3 +} \ No newline at end of file diff --git a/data/e3_ksubspace/EleutherAI--pythia-1b_d1000.json b/data/e3_ksubspace/EleutherAI--pythia-1b_d1000.json new file mode 100644 index 0000000000000000000000000000000000000000..0aa165aaee3535d7eda76ccfc350058645fd49af --- /dev/null +++ b/data/e3_ksubspace/EleutherAI--pythia-1b_d1000.json @@ -0,0 +1,42 @@ +{ + "model": "pythia-1b", + "n_layers": 16, + "d_model": 2048, + "d_head": 256, + "theta": 10000, + "seed": 42, + "dist_long": 1000, + "dist_short": 10, + "l_crit": 15, + "n_prompts": 200, + "k_99pct_variance": 182, + "k_90pct_variance": 98, + "k_50pct_variance": 1, + "top_singular_value": 880.2179565429688, + "spectral_gap_ratio": 4.608410835266113, + "cumvar_at_k99": 0.990045964717865, + "cumvar_at_k90": 0.900945246219635, + "cumvar_at_k50": 0.5227205157279968, + "singular_values_top10": [ + 880.2179565429688, + 191.00250244140625, + 178.61480712890625, + 132.24012756347656, + 122.12053680419922, + 117.49409484863281, + 110.08806610107422, + 108.31230163574219, + 105.69137573242188, + 103.90699768066406 + ], + "k_pred": 140.91648842508062, + "k_pred_int": 141, + "k_pred_formula": "d_head x log_theta(d_long/2pi) = 256 x 0.5505", + "k_error_pct": 29.154509904468537, + "logp_gap": 2.0964049315452584, + "R_full_avg": 0.9502962040770896, + "R_k99_avg": 0.898072703256963, + "partial_recovery_pct": 94.50450284910428, + "ratio_k_dmodel": 0.0888671875, + "runtime_seconds": 1857.0458045005798 +} \ No newline at end of file diff --git a/data/e3_ksubspace/EleutherAI--pythia-70m_d1000.json b/data/e3_ksubspace/EleutherAI--pythia-70m_d1000.json new file mode 100644 index 0000000000000000000000000000000000000000..0556c0242d728064662316879b9f01b8fb67e23c --- /dev/null +++ b/data/e3_ksubspace/EleutherAI--pythia-70m_d1000.json @@ -0,0 +1,42 @@ +{ + "model": "pythia-70m", + "n_layers": 6, + "d_model": 512, + "d_head": 64, + "theta": 10000, + "seed": 42, + "dist_long": 1000, + "dist_short": 10, + "l_crit": 4, + "n_prompts": 200, + "k_99pct_variance": 64, + "k_90pct_variance": 1, + "k_50pct_variance": 1, + "top_singular_value": 742.4482421875, + "spectral_gap_ratio": 8.484195709228516, + "cumvar_at_k99": 0.9901030659675598, + "cumvar_at_k90": 0.9406560063362122, + "cumvar_at_k50": 0.9406560063362122, + "singular_values_top10": [ + 742.4482421875, + 87.50955963134766, + 57.6650505065918, + 41.584407806396484, + 36.161293029785156, + 28.921112060546875, + 28.18744659423828, + 24.050268173217773, + 21.590150833129883, + 21.302764892578125 + ], + "k_pred": 35.229122106270154, + "k_pred_int": 35, + "k_pred_formula": "d_head x log_theta(d_long/2pi) = 64 x 0.5505", + "k_error_pct": 81.66788206342828, + "logp_gap": 1.1474852193891998, + "R_full_avg": 1.302688409976077, + "R_k99_avg": 1.268225332676965, + "partial_recovery_pct": 97.35446504051227, + "ratio_k_dmodel": 0.125, + "runtime_seconds": 226.89268803596497 +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--gpt-j-6B_mongo.json b/data/e4_gamma/EleutherAI--gpt-j-6B_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..950ab67f5a76b6c8f8423f99b0a61928476cfc2b --- /dev/null +++ b/data/e4_gamma/EleutherAI--gpt-j-6B_mongo.json @@ -0,0 +1,123 @@ +{ + "model": "EleutherAI/gpt-j-6B", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.00839791604421205, + "std_across_seeds": 0.00019323688493508385, + "per_seed_means": [ + 0.008127806646904597, + 0.008497034331473212, + 0.008568907154258341 + ] + }, + "20": { + "mean_across_seeds": 0.00509554898715578, + "std_across_seeds": 0.0004143109313167893, + "per_seed_means": [ + 0.0045346545067150145, + 0.005229270776035264, + 0.005522721678717062 + ] + }, + "30": { + "mean_across_seeds": 0.0032989257112947395, + "std_across_seeds": 0.0001286165676687711, + "per_seed_means": [ + 0.003299940403861304, + 0.0031408983344833057, + 0.0034559383955396093 + ] + }, + "50": { + "mean_across_seeds": 0.0022861401645544294, + "std_across_seeds": 0.00015427519653425807, + "per_seed_means": [ + 0.002121648237807676, + 0.002244256925381099, + 0.0024925153304745135 + ] + }, + "100": { + "mean_across_seeds": 0.0015001902751909156, + "std_across_seeds": 0.0002457887830504289, + "per_seed_means": [ + 0.0013760252240657186, + 0.0012811047569266521, + 0.001843440844580376 + ] + }, + "200": { + "mean_across_seeds": 0.0006473004045053838, + "std_across_seeds": 5.715937265321037e-05, + "per_seed_means": [ + 0.0006406685622278019, + 0.0005808466701758638, + 0.0007203859811124858 + ] + }, + "500": { + "mean_across_seeds": 0.00023765665108536874, + "std_across_seeds": 3.906551588796154e-05, + "per_seed_means": [ + 0.0001958514337457018, + 0.00022727963939663217, + 0.0002898388801137723 + ] + }, + "1000": { + "mean_across_seeds": 0.00017106029921908178, + "std_across_seeds": 2.5879584004771193e-05, + "per_seed_means": [ + 0.000134487032743588, + 0.00018815259712937405, + 0.00019054126778428326 + ] + } + }, + "fit_power_law": { + "gamma": 0.8967966648253523, + "log_A": -2.5750187061655745, + "R2": 0.98687, + "n_points": 6 + }, + "fit_exponential": { + "lambda": 0.002903210789879725, + "log_A": -6.200008987171874, + "R2": 0.813627, + "n_points": 6 + }, + "delta_R2_power_minus_exp": 0.1732, + "gamma_CI_95": { + "lo": 0.7975638000898441, + "hi": 1.036499726945172 + }, + "decision": "UNCLEAR: \u03b3=0.897 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--gpt-j-6B_random.json b/data/e4_gamma/EleutherAI--gpt-j-6B_random.json new file mode 100644 index 0000000000000000000000000000000000000000..ae5176f3f91b5300f9bcff2bd7cffe2ffd8276c3 --- /dev/null +++ b/data/e4_gamma/EleutherAI--gpt-j-6B_random.json @@ -0,0 +1,123 @@ +{ + "model": "EleutherAI/gpt-j-6B", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.00790414453121937, + "std_across_seeds": 0.00013863079948421702, + "per_seed_means": [ + 0.007774174048875769, + 0.008096245873409014, + 0.007842013671373327 + ] + }, + "20": { + "mean_across_seeds": 0.005052289131304456, + "std_across_seeds": 0.00015772763747498123, + "per_seed_means": [ + 0.0048292399616912006, + 0.005161861182811359, + 0.005165766249410808 + ] + }, + "30": { + "mean_across_seeds": 0.0033700674681717327, + "std_across_seeds": 0.00015998166115533112, + "per_seed_means": [ + 0.0032935648031222325, + 0.003223923289527496, + 0.0035927143118654687 + ] + }, + "50": { + "mean_across_seeds": 0.0029467191888640325, + "std_across_seeds": 5.830230097669026e-05, + "per_seed_means": [ + 0.0030064180439027645, + 0.0028676176296236616, + 0.0029661218930656713 + ] + }, + "100": { + "mean_across_seeds": 0.0018710899210741948, + "std_across_seeds": 5.4059302628310255e-05, + "per_seed_means": [ + 0.001803609950778385, + 0.0018737099366262556, + 0.0019359498758179446 + ] + }, + "200": { + "mean_across_seeds": 0.0008449158267143907, + "std_across_seeds": 4.219505062090254e-05, + "per_seed_means": [ + 0.0008908263925695792, + 0.0007889484489957491, + 0.0008549726385778437 + ] + }, + "500": { + "mean_across_seeds": 0.00032950418016601665, + "std_across_seeds": 1.2120073367659681e-05, + "per_seed_means": [ + 0.0003462403745894941, + 0.00031793153699254616, + 0.00032434062891600966 + ] + }, + "1000": { + "mean_across_seeds": 0.00022974714450861534, + "std_across_seeds": 1.0465813898187747e-05, + "per_seed_means": [ + 0.00024454623431665825, + 0.00022254784062776404, + 0.00022214735858142376 + ] + } + }, + "fit_power_law": { + "gamma": 0.8347553760849855, + "log_A": -2.6580238264494502, + "R2": 0.980141, + "n_points": 6 + }, + "fit_exponential": { + "lambda": 0.002744567753253188, + "log_A": -6.019009744982984, + "R2": 0.833517, + "n_points": 6 + }, + "delta_R2_power_minus_exp": 0.1466, + "gamma_CI_95": { + "lo": 0.6805660295164746, + "hi": 0.9949392932271522 + }, + "decision": "UNCLEAR: \u03b3=0.835 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-1.4b_mongo.json b/data/e4_gamma/EleutherAI--pythia-1.4b_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..dc44252f7dd074b88e6cc9025d757303bce7adfd --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-1.4b_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-1.4b", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.00930954304847142, + "std_across_seeds": 0.00046598500655753905, + "per_seed_means": [ + 0.009433638171758503, + 0.008686992730557298, + 0.009807998243098458 + ] + }, + "20": { + "mean_across_seeds": 0.006164806057543805, + "std_across_seeds": 0.00014470988657920404, + "per_seed_means": [ + 0.006355531241279095, + 0.006005183407105505, + 0.006133703524246812 + ] + }, + "30": { + "mean_across_seeds": 0.003750873178942129, + "std_across_seeds": 0.00046433891808706455, + "per_seed_means": [ + 0.0035096941760275513, + 0.0033425104493896166, + 0.004400414911409219 + ] + }, + "50": { + "mean_across_seeds": 0.0024919367112710865, + "std_across_seeds": 0.00020655362634590453, + "per_seed_means": [ + 0.0024369027153200782, + 0.002271008453099057, + 0.0027678989653941244 + ] + }, + "100": { + "mean_across_seeds": 0.0015481134017722475, + "std_across_seeds": 9.321839775341404e-05, + "per_seed_means": [ + 0.0015698801611627763, + 0.001424628242966719, + 0.001649831801187247 + ] + }, + "200": { + "mean_across_seeds": 0.0008042571904176535, + "std_across_seeds": 0.00011442387728512982, + "per_seed_means": [ + 0.0007252432114910334, + 0.0007214660650546042, + 0.0009660622947073231 + ] + }, + "500": { + "mean_across_seeds": 0.00021084786865887712, + "std_across_seeds": 1.56926991859727e-05, + "per_seed_means": [ + 0.000196650760663033, + 0.00020317405639313317, + 0.00023271878892046515 + ] + }, + "1000": { + "mean_across_seeds": 0.00018452253824863065, + "std_across_seeds": 9.089897272486009e-06, + "per_seed_means": [ + 0.00019737111651920713, + 0.0001777448711921655, + 0.00017845162703451932 + ] + }, + "2000": { + "mean_across_seeds": 0.00036662833303530027, + "std_across_seeds": 6.854456752695853e-05, + "per_seed_means": [ + 0.0004429141562529064, + 0.00027668878554929204, + 0.00038028205730370243 + ] + } + }, + "fit_power_law": { + "gamma": 0.7050725013322717, + "log_A": -3.342801964634624, + "R2": 0.841258, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0010766062690250938, + "log_A": -6.567544805679143, + "R2": 0.422707, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.4186, + "gamma_CI_95": { + "lo": 0.49742183078097757, + "hi": 1.0319364818957766 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-1.4b_random.json b/data/e4_gamma/EleutherAI--pythia-1.4b_random.json new file mode 100644 index 0000000000000000000000000000000000000000..3ea664be4078ce0d627b9d65260ca629535b0d92 --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-1.4b_random.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-1.4b", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.00868701820054816, + "std_across_seeds": 0.000225376018293542, + "per_seed_means": [ + 0.00849575198100259, + 0.008561847213034829, + 0.009003455407607058 + ] + }, + "20": { + "mean_across_seeds": 0.007241151898374988, + "std_across_seeds": 0.000907916062609691, + "per_seed_means": [ + 0.008468000154631833, + 0.006299703560459117, + 0.006955751980034014 + ] + }, + "30": { + "mean_across_seeds": 0.004176199407730666, + "std_across_seeds": 0.0003113188967399462, + "per_seed_means": [ + 0.003928308213750522, + 0.0039850392285734415, + 0.004615250780868034 + ] + }, + "50": { + "mean_across_seeds": 0.0029845095390919596, + "std_across_seeds": 0.00023642395319887298, + "per_seed_means": [ + 0.0027204082436704385, + 0.0029389867997573066, + 0.0032941335738481334 + ] + }, + "100": { + "mean_across_seeds": 0.0028477065866657843, + "std_across_seeds": 0.0010192211252852063, + "per_seed_means": [ + 0.004288771225837991, + 0.002153952665782223, + 0.002100395868377139 + ] + }, + "200": { + "mean_across_seeds": 0.0014213456220588544, + "std_across_seeds": 0.00018510584610857393, + "per_seed_means": [ + 0.0012832466786494479, + 0.0012978002527961508, + 0.0016829899347309645 + ] + }, + "500": { + "mean_across_seeds": 0.0004443507219710025, + "std_across_seeds": 2.1944710361610797e-05, + "per_seed_means": [ + 0.0004357509139420775, + 0.0004228264523165611, + 0.00047447479965436894 + ] + }, + "1000": { + "mean_across_seeds": 0.00037049855294753794, + "std_across_seeds": 7.606907661861908e-06, + "per_seed_means": [ + 0.0003622182499869571, + 0.0003686910155132258, + 0.00038058639334243103 + ] + }, + "2000": { + "mean_across_seeds": 0.0003167598478871191, + "std_across_seeds": 8.808747176569926e-06, + "per_seed_means": [ + 0.00031430382912124816, + 0.0003074111366489281, + 0.0003285645778911809 + ] + } + }, + "fit_power_law": { + "gamma": 0.6875846707476845, + "log_A": -3.0428636808858562, + "R2": 0.948837, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0012564601985353926, + "log_A": -6.073132081889184, + "R2": 0.682811, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.266, + "gamma_CI_95": { + "lo": 0.5680458803652685, + "hi": 0.8957392897879572 + }, + "decision": "UNCLEAR: \u03b3=0.688 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-14m_mongo.json b/data/e4_gamma/EleutherAI--pythia-14m_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..97c1a392c6e62d37e5851c2f05731ed6478eb751 --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-14m_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-14m", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.012390103595874584, + "std_across_seeds": 0.00146593819912507, + "per_seed_means": [ + 0.01203493590466678, + 0.010798830709342535, + 0.014336544173614433 + ] + }, + "20": { + "mean_across_seeds": 0.0069860522150864, + "std_across_seeds": 0.0002394408105901627, + "per_seed_means": [ + 0.006651017426047474, + 0.007111006476916373, + 0.007196132742295352 + ] + }, + "30": { + "mean_across_seeds": 0.004465355200259688, + "std_across_seeds": 0.0006985250334266525, + "per_seed_means": [ + 0.0039960742109299945, + 0.003947176096165398, + 0.005452815293683671 + ] + }, + "50": { + "mean_across_seeds": 0.0032919485721827693, + "std_across_seeds": 0.0004476447998581526, + "per_seed_means": [ + 0.0034145738973165862, + 0.0026927687582792713, + 0.00376850306095245 + ] + }, + "100": { + "mean_across_seeds": 0.0011803068259986403, + "std_across_seeds": 0.0004023732892548156, + "per_seed_means": [ + 0.000915640011880896, + 0.0008763834233832313, + 0.0017488970427317933 + ] + }, + "200": { + "mean_across_seeds": 0.000789741734192325, + "std_across_seeds": 9.65043332514401e-05, + "per_seed_means": [ + 0.0007327324370483742, + 0.000710858839102002, + 0.0009256339264265989 + ] + }, + "500": { + "mean_across_seeds": 0.00044898551188351935, + "std_across_seeds": 0.00031459565482308744, + "per_seed_means": [ + 0.0003210191007049919, + 0.00014395098448706753, + 0.0008819864504584985 + ] + }, + "1000": { + "mean_across_seeds": 0.00022560695875026605, + "std_across_seeds": 0.00016511484107135197, + "per_seed_means": [ + 0.00024422961300868927, + 1.4716208875812005e-05, + 0.0004178750543662968 + ] + }, + "2000": { + "mean_across_seeds": 0.0003564109894130432, + "std_across_seeds": 0.0003314753614780148, + "per_seed_means": [ + 0.00026265843942539834, + 5.516328529718469e-06, + 0.0008010582002840128 + ] + } + }, + "fit_power_law": { + "gamma": 0.6852875452461592, + "log_A": -3.294048319265473, + "R2": 0.904735, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.001106557451848494, + "log_A": -6.394955259676772, + "R2": 0.508379, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.3964, + "gamma_CI_95": { + "lo": 0.4357537943339858, + "hi": 0.9132980416812998 + }, + "decision": "UNCLEAR: \u03b3=0.685 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-14m_random.json b/data/e4_gamma/EleutherAI--pythia-14m_random.json new file mode 100644 index 0000000000000000000000000000000000000000..6af2351c8fdd4b0340348dd962a1c80ac4254dcc --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-14m_random.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-14m", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.010373582980052257, + "std_across_seeds": 0.0006252837496852004, + "per_seed_means": [ + 0.010257982229813934, + 0.009672142285853624, + 0.011190624424489214 + ] + }, + "20": { + "mean_across_seeds": 0.006793614017466704, + "std_across_seeds": 0.0004633287803798598, + "per_seed_means": [ + 0.006414954091111819, + 0.006519831863309567, + 0.007446056097978726 + ] + }, + "30": { + "mean_across_seeds": 0.005060315049162858, + "std_across_seeds": 0.0007549517281669201, + "per_seed_means": [ + 0.006126645363353115, + 0.004573333369335159, + 0.004480966414800302 + ] + }, + "50": { + "mean_across_seeds": 0.0042915507254656406, + "std_across_seeds": 0.00044407960367519523, + "per_seed_means": [ + 0.0036970104288775473, + 0.004413602671508367, + 0.004764039076011007 + ] + }, + "100": { + "mean_across_seeds": 0.0019918971036935952, + "std_across_seeds": 0.0001427226381169157, + "per_seed_means": [ + 0.0021524308461327263, + 0.0020175845400808613, + 0.0018056759248671977 + ] + }, + "200": { + "mean_across_seeds": 0.0011028067811053512, + "std_across_seeds": 0.00022419152634048253, + "per_seed_means": [ + 0.0014089059133524037, + 0.0010213131487156108, + 0.0008782012812480389 + ] + }, + "500": { + "mean_across_seeds": 0.00045950785397306186, + "std_across_seeds": 0.00010281434631793879, + "per_seed_means": [ + 0.0005975673015685364, + 0.00035097020172543126, + 0.000429986058625218 + ] + }, + "1000": { + "mean_across_seeds": 0.00012169795030211442, + "std_across_seeds": 4.121883822012145e-05, + "per_seed_means": [ + 0.00010464097544797824, + 0.00017849944926714065, + 8.195342619122433e-05 + ] + }, + "2000": { + "mean_across_seeds": 0.00010627185478203357, + "std_across_seeds": 5.407436262983947e-05, + "per_seed_means": [ + 0.00017853263377891912, + 4.8465674529675575e-05, + 9.181725603750598e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.003714187534367, + "log_A": -1.6477058895472607, + "R2": 0.977698, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0019252473876952617, + "log_A": -6.020695902400171, + "R2": 0.775214, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2025, + "gamma_CI_95": { + "lo": 0.8529431527599634, + "hi": 1.1837539173829756 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-160m_mongo.json b/data/e4_gamma/EleutherAI--pythia-160m_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..17740e5b6c74332c8913b3ca301ff7fce767e044 --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-160m_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-160m", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.01184501802885077, + "std_across_seeds": 0.001378859479481131, + "per_seed_means": [ + 0.011957902879609416, + 0.010102656579886874, + 0.013474494627056022 + ] + }, + "20": { + "mean_across_seeds": 0.00870004014771742, + "std_across_seeds": 0.0006360259839224177, + "per_seed_means": [ + 0.008050499407496924, + 0.008485954137674223, + 0.009563666897981117 + ] + }, + "30": { + "mean_across_seeds": 0.004389970480309179, + "std_across_seeds": 0.0011407267671031584, + "per_seed_means": [ + 0.0036179631628328935, + 0.003549234614086648, + 0.006002713664007994 + ] + }, + "50": { + "mean_across_seeds": 0.003499219432576663, + "std_across_seeds": 0.000770145930177433, + "per_seed_means": [ + 0.0031687027386700112, + 0.002765725217953635, + 0.0045632303411063425 + ] + }, + "100": { + "mean_across_seeds": 0.0016630664256485437, + "std_across_seeds": 0.0005458147513655124, + "per_seed_means": [ + 0.001334343857500547, + 0.0012225916004293444, + 0.0024322638190157403 + ] + }, + "200": { + "mean_across_seeds": 0.0011387520531247395, + "std_across_seeds": 0.0006016585561184202, + "per_seed_means": [ + 0.0007571180729428306, + 0.0006709673759663322, + 0.0019881707104650558 + ] + }, + "500": { + "mean_across_seeds": 0.0007303262313851318, + "std_across_seeds": 0.0006923605369903511, + "per_seed_means": [ + 0.00026335111497852875, + 0.00021849818511933942, + 0.001709129394057527 + ] + }, + "1000": { + "mean_across_seeds": 0.0005382596583993088, + "std_across_seeds": 0.0006345797008602084, + "per_seed_means": [ + 9.761169005287229e-05, + 8.152447151739276e-05, + 0.0014356428136276615 + ] + }, + "2000": { + "mean_across_seeds": 0.0006330042882872375, + "std_across_seeds": 0.0008601009017326663, + "per_seed_means": [ + 2.6749389705855718e-05, + 2.289486239078542e-05, + 0.0018493686127650714 + ] + } + }, + "fit_power_law": { + "gamma": 0.510891565491545, + "log_A": -3.8559388637995884, + "R2": 0.916873, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0008260835489515251, + "log_A": -6.167082924869525, + "R2": 0.516611, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.4003, + "gamma_CI_95": { + "lo": 0.32767446550421003, + "hi": 0.7113451846826664 + }, + "decision": "UNCLEAR: \u03b3=0.511 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-160m_random.json b/data/e4_gamma/EleutherAI--pythia-160m_random.json new file mode 100644 index 0000000000000000000000000000000000000000..8c35a3ad7c8bf3305b69038ad8e6831f36772fea --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-160m_random.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-160m", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.010829462509912749, + "std_across_seeds": 0.0004933662521940574, + "per_seed_means": [ + 0.010583337599722048, + 0.010387120433151723, + 0.011517929496864478 + ] + }, + "20": { + "mean_across_seeds": 0.009873490159192847, + "std_across_seeds": 0.0006517332751691898, + "per_seed_means": [ + 0.010788665946262578, + 0.009321169209045669, + 0.009510635322270294 + ] + }, + "30": { + "mean_across_seeds": 0.003808915811306279, + "std_across_seeds": 0.00010559975526077366, + "per_seed_means": [ + 0.003957300891634077, + 0.003720116306406756, + 0.0037493302358780054 + ] + }, + "50": { + "mean_across_seeds": 0.003134424632622136, + "std_across_seeds": 0.00011024204387092253, + "per_seed_means": [ + 0.0030553750837376964, + 0.003290325259246553, + 0.003057573554882159 + ] + }, + "100": { + "mean_across_seeds": 0.001589946753811091, + "std_across_seeds": 0.00010627266733168153, + "per_seed_means": [ + 0.0016439523688556315, + 0.0016844073765484306, + 0.0014414805160292113 + ] + }, + "200": { + "mean_across_seeds": 0.0009230124831406607, + "std_across_seeds": 0.00013459768567844406, + "per_seed_means": [ + 0.0011075238784542308, + 0.0008712717732608629, + 0.0007902417977068884 + ] + }, + "500": { + "mean_across_seeds": 0.0003505013234017598, + "std_across_seeds": 4.860340884797636e-05, + "per_seed_means": [ + 0.00041482529151835477, + 0.0003393216502687816, + 0.000297357028418143 + ] + }, + "1000": { + "mean_across_seeds": 0.00015388178632242167, + "std_across_seeds": 8.562391442141839e-06, + "per_seed_means": [ + 0.00015148034889231592, + 0.00014480404968101842, + 0.00016536096039393062 + ] + }, + "2000": { + "mean_across_seeds": 5.142585230411594e-05, + "std_across_seeds": 9.118662101140663e-06, + "per_seed_means": [ + 3.9784371095568836e-05, + 6.205095907110566e-05, + 5.24422267456733e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.0171452847779678, + "log_A": -1.8268598516498322, + "R2": 0.981723, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.002070284850238622, + "log_A": -6.192254167723398, + "R2": 0.87649, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.1052, + "gamma_CI_95": { + "lo": 0.8278231891480886, + "hi": 1.160612970889765 + }, + "decision": "CONFIRMED: \u03b3 law holds (\u03b3\u00d7ln(\u03b8) = C)" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-1b_layerwise.json b/data/e4_gamma/EleutherAI--pythia-1b_layerwise.json new file mode 100644 index 0000000000000000000000000000000000000000..a218815ee208b0fa2c8e2a85477511215faad29b --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-1b_layerwise.json @@ -0,0 +1,313 @@ +{ + "model": "EleutherAI/pythia-1b", + "theta": 10000, + "d_head": 256, + "n_layers": 16, + "L_crit_known": 15, + "z": 0.282842712474619, + "gamma_pos_pred": 0.9925355840547238, + "gamma_pade": 0.7522013138014093, + "beta_gamma": 0.2, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "n_prompts": 150, + "seeds": [ + 42, + 123, + 7 + ], + "layer_gamma": { + "0": { + "gamma": 1.13879191664383, + "R2": 0.989101, + "n_points": 7, + "type": "pos" + }, + "1": { + "gamma": 0.7357707192097281, + "R2": 0.773671, + "n_points": 7, + "type": "pos" + }, + "2": { + "gamma": 0.9771238359273494, + "R2": 0.884293, + "n_points": 7, + "type": "pos" + }, + "3": { + "gamma": 0.8089874794887668, + "R2": 0.923742, + "n_points": 7, + "type": "pos" + }, + "4": { + "gamma": 0.527133382907792, + "R2": 0.917358, + "n_points": 7, + "type": "pos" + }, + "5": { + "gamma": 0.5026001519708836, + "R2": 0.926494, + "n_points": 7, + "type": "pos" + }, + "6": { + "gamma": 0.6356504608471039, + "R2": 0.950752, + "n_points": 7, + "type": "pos" + }, + "7": { + "gamma": 0.7379019578605543, + "R2": 0.853502, + "n_points": 7, + "type": "pos" + }, + "8": { + "gamma": 0.814899451520626, + "R2": 0.912246, + "n_points": 7, + "type": "pos" + }, + "9": { + "gamma": 0.7097288225189304, + "R2": 0.949197, + "n_points": 7, + "type": "pos" + }, + "10": { + "gamma": 0.6208513029427555, + "R2": 0.78962, + "n_points": 7, + "type": "pos" + }, + "11": { + "gamma": 0.6698224233704795, + "R2": 0.968785, + "n_points": 7, + "type": "pos" + }, + "12": { + "gamma": 0.6055081734453818, + "R2": 0.947673, + "n_points": 7, + "type": "pos" + }, + "13": { + "gamma": 0.6971249845232657, + "R2": 0.917616, + "n_points": 7, + "type": "pos" + }, + "14": { + "gamma": 0.8103367929633055, + "R2": 0.94372, + "n_points": 7, + "type": "pos" + }, + "15": { + "gamma": 0.8528983955699013, + "R2": 0.935571, + "n_points": 7, + "type": "sem" + } + }, + "per_layer_agg": { + "0": { + "10": 0.023287173575825163, + "20": 0.012761972430679535, + "30": 0.008739391267299652, + "50": 0.005774891078472137, + "100": 0.003097074362966749, + "200": 0.001237770782576667, + "500": 0.0002983309825261434, + "1000": 0.0001609127057923211, + "2000": 0.00010039887494511074 + }, + "1": { + "10": 0.014171950005822711, + "20": 0.008971484220690197, + "30": 0.0051612050500180985, + "50": 0.007755267272392907, + "100": 0.0033257472018400833, + "200": 0.000814016345474455, + "500": 0.00038598226176367865, + "1000": 0.00029986533853742814, + "2000": 0.0006743205421500736 + }, + "2": { + "10": 0.01453488290309906, + "20": 0.00871283976568116, + "30": 0.0050693359805477995, + "50": 0.005772705343034532, + "100": 0.002663780417707231, + "200": 0.0006871984236770205, + "500": 0.00022906308372815448, + "1000": 0.0005532768368721008, + "2000": 7.777710755666098e-05 + }, + "3": { + "10": 0.012952246665954589, + "20": 0.009007836911413407, + "30": 0.005951569527387619, + "50": 0.00468554933865865, + "100": 0.004074350115325716, + "200": 0.0012779432700739965, + "500": 0.0003548884060647752, + "1000": 0.0005072926647133297, + "2000": 0.00026480979389614524 + }, + "4": { + "10": 0.009142524898052217, + "20": 0.006767975141604742, + "30": 0.0037466619246535834, + "50": 0.0030552635259098477, + "100": 0.0031027018692758347, + "200": 0.001693786382675171, + "500": 0.000837334394454956, + "1000": 0.0009975386824872757, + "2000": 0.000359780920876397 + }, + "5": { + "10": 0.01093157473537657, + "20": 0.0076380522052447, + "30": 0.004587366282939911, + "50": 0.0033414626121520997, + "100": 0.0021655780242549045, + "200": 0.0017234790159596338, + "500": 0.0007033846775690715, + "1000": 0.0010454465779993268, + "2000": 0.000491521590285831 + }, + "6": { + "10": 0.010107053137487837, + "20": 0.006772236492898729, + "30": 0.0038019116057289976, + "50": 0.002806223995155758, + "100": 0.0020480300651656257, + "200": 0.0010693785382641684, + "500": 0.0004214300380812751, + "1000": 0.0005711258285575443, + "2000": 0.00026035734348826936 + }, + "7": { + "10": 0.00790486506289906, + "20": 0.005169999450445175, + "30": 0.0036405806408988106, + "50": 0.00276623730858167, + "100": 0.002493643512328466, + "200": 0.001324114633931054, + "500": 0.0006054121255874633, + "1000": 0.0007640798555480109, + "2000": 9.761429495281643e-05 + }, + "8": { + "10": 0.008711223834090764, + "20": 0.007626397957404454, + "30": 0.0056268137031131316, + "50": 0.004460600266853969, + "100": 0.0031554626921812697, + "200": 0.0013908600476053026, + "500": 0.0006027436090840234, + "1000": 0.0008089686764611139, + "2000": 0.00012790492839283415 + }, + "9": { + "10": 0.009790150059594048, + "20": 0.008512286957767274, + "30": 0.005793256610631942, + "50": 0.0041007401545842485, + "100": 0.003971491638157103, + "200": 0.0014884891112645468, + "500": 0.0006241897410816616, + "1000": 0.0007302767866187625, + "2000": 0.0002907270358668433 + }, + "10": { + "10": 0.005038524601194594, + "20": 0.004626353863212797, + "30": 0.002466487238804499, + "50": 0.0023082222044467924, + "100": 0.002587024056249195, + "200": 0.0011542993618382348, + "500": 0.0006628256042798359, + "1000": 0.0008455400168895722, + "2000": 0.00011965241697099473 + }, + "11": { + "10": 0.008719167593452665, + "20": 0.007992720289362801, + "30": 0.006105314675304625, + "50": 0.004175959279139837, + "100": 0.0037570750216643013, + "200": 0.0018183539642228023, + "500": 0.0008812188605467477, + "1000": 0.0008290305071406894, + "2000": 0.0003235883845223321 + }, + "12": { + "10": 0.008280009395546383, + "20": 0.007897668017281426, + "30": 0.00534157062570254, + "50": 0.003976477500465181, + "100": 0.004049868418110742, + "200": 0.002005085481537713, + "500": 0.001032156530353758, + "1000": 0.0009831561810440487, + "2000": 0.0003697805106639862 + }, + "13": { + "10": 0.01029373879233996, + "20": 0.007694267200099097, + "30": 0.004842915882666905, + "50": 0.0045533665186829044, + "100": 0.003288008454773161, + "200": 0.001259168138106664, + "500": 0.000650797039270401, + "1000": 0.0008947557873196072, + "2000": 0.00022810177670584786 + }, + "14": { + "10": 0.01466528390844663, + "20": 0.012698218176762264, + "30": 0.0072026044958167605, + "50": 0.006098132994439866, + "100": 0.003323908862140444, + "200": 0.001219436013036304, + "500": 0.0004505762457847595, + "1000": 0.0006288045644760132, + "2000": 0.0002749601337644789 + }, + "15": { + "10": 0.01335475915008121, + "20": 0.009260030653741624, + "30": 0.006401976711220211, + "50": 0.004598343438572354, + "100": 0.0026834943393866223, + "200": 0.0008934098813268873, + "500": 0.0005171574486626519, + "1000": 0.0006370346082581415, + "2000": 0.00013038047485881382 + } + } +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-1b_mongo.json b/data/e4_gamma/EleutherAI--pythia-1b_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..4c55476414247bd1d1796610f0ba6ab1e0555c8a --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-1b_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-1b", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.010958388174573582, + "std_across_seeds": 0.0004834559168031873, + "per_seed_means": [ + 0.011297275526449085, + 0.010274686822667719, + 0.011303202174603939 + ] + }, + "20": { + "mean_across_seeds": 0.006365462491909664, + "std_across_seeds": 4.797004634512219e-05, + "per_seed_means": [ + 0.006431558985883991, + 0.006319180646290382, + 0.006345647843554616 + ] + }, + "30": { + "mean_across_seeds": 0.003956688532812728, + "std_across_seeds": 0.00041444934898881186, + "per_seed_means": [ + 0.003707738428687056, + 0.003621630910784006, + 0.0045406962589671215 + ] + }, + "50": { + "mean_across_seeds": 0.0027740057195640272, + "std_across_seeds": 0.00017409472550222068, + "per_seed_means": [ + 0.0028004927871127924, + 0.0025487780198454857, + 0.0029727463517338036 + ] + }, + "100": { + "mean_across_seeds": 0.0014326052833348512, + "std_across_seeds": 4.960638948465883e-05, + "per_seed_means": [ + 0.0014700610004365444, + 0.001362506297106544, + 0.0014652485524614652 + ] + }, + "200": { + "mean_across_seeds": 0.0006671193842258718, + "std_across_seeds": 0.00014972843939483376, + "per_seed_means": [ + 0.00054337820969522, + 0.0005801813707997401, + 0.0008777985721826554 + ] + }, + "500": { + "mean_across_seeds": 0.00020134633510477014, + "std_across_seeds": 1.977215748224994e-05, + "per_seed_means": [ + 0.00018427866821487746, + 0.00019069870933890342, + 0.0002290616277605295 + ] + }, + "1000": { + "mean_across_seeds": 0.00015050364037354788, + "std_across_seeds": 1.545945974564147e-05, + "per_seed_means": [ + 0.00014964959894617398, + 0.00016985010355710984, + 0.0001320112186173598 + ] + }, + "2000": { + "mean_across_seeds": 0.00010232122304538885, + "std_across_seeds": 6.657508586021642e-06, + "per_seed_means": [ + 0.00010441893401245276, + 0.00010922116227447987, + 9.332357284923395e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.9311078627189842, + "log_A": -2.350543685121484, + "R2": 0.983104, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.001683695888726774, + "log_A": -6.4638962346156505, + "R2": 0.692773, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2903, + "gamma_CI_95": { + "lo": 0.8649963102497168, + "hi": 1.0919504310156105 + }, + "decision": "CONFIRMED: \u03b3 law holds (\u03b3\u00d7ln(\u03b8) = C)" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-1b_random.json b/data/e4_gamma/EleutherAI--pythia-1b_random.json new file mode 100644 index 0000000000000000000000000000000000000000..b6ab579f2b0210a580951c077b97814d2c9cd2fd --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-1b_random.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-1b", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.011367820525127982, + "std_across_seeds": 0.0003061832409116484, + "per_seed_means": [ + 0.011192221554617087, + 0.01111284309066832, + 0.011798396930098534 + ] + }, + "20": { + "mean_across_seeds": 0.008256896255123948, + "std_across_seeds": 0.00032382404546097755, + "per_seed_means": [ + 0.00865409774084886, + 0.008255694229155779, + 0.007860896795367202 + ] + }, + "30": { + "mean_across_seeds": 0.005279935139955745, + "std_across_seeds": 0.00015508039193542144, + "per_seed_means": [ + 0.005486616104220351, + 0.0051130562741309405, + 0.005240133041515946 + ] + }, + "50": { + "mean_across_seeds": 0.004389340176971422, + "std_across_seeds": 0.00015301220161172737, + "per_seed_means": [ + 0.0041992794442921874, + 0.004394778193285068, + 0.004573962893337011 + ] + }, + "100": { + "mean_across_seeds": 0.003111702454172903, + "std_across_seeds": 0.0003438079611532678, + "per_seed_means": [ + 0.003580885141467055, + 0.0029875944834202527, + 0.0027666277376314006 + ] + }, + "200": { + "mean_across_seeds": 0.0013160493473211923, + "std_across_seeds": 0.00021701703531491198, + "per_seed_means": [ + 0.0014770981731514136, + 0.0014617815303305784, + 0.0010092683384815852 + ] + }, + "500": { + "mean_across_seeds": 0.0005785931936568684, + "std_across_seeds": 0.00014003201332155925, + "per_seed_means": [ + 0.00038075152474145095, + 0.0006850937816003958, + 0.0006699342746287584 + ] + }, + "1000": { + "mean_across_seeds": 0.0007035690980652969, + "std_across_seeds": 0.00010519359687093437, + "per_seed_means": [ + 0.00057055722301205, + 0.0008277761625746886, + 0.0007123739086091519 + ] + }, + "2000": { + "mean_across_seeds": 0.00026197975811858974, + "std_across_seeds": 9.881774537676763e-06, + "per_seed_means": [ + 0.0002482394129037857, + 0.00026664167642593384, + 0.0002710581850260496 + ] + } + }, + "fit_power_law": { + "gamma": 0.7127537531683527, + "log_A": -2.713966296451458, + "R2": 0.956201, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0013622326398466406, + "log_A": -5.822022876702345, + "R2": 0.752726, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2035, + "gamma_CI_95": { + "lo": 0.5792643838884316, + "hi": 0.8495977917640387 + }, + "decision": "UNCLEAR: \u03b3=0.713 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-2.8b_mongo.json b/data/e4_gamma/EleutherAI--pythia-2.8b_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..acbe4be2d5100485b272e6787d38518b93f51d93 --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-2.8b_mongo.json @@ -0,0 +1,90 @@ +{ + "model": "EleutherAI/pythia-2.8b", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100 + ], + "distances_fit": [ + 30, + 50, + 100 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.008272597978745277, + "std_across_seeds": 0.0012549327039177746, + "per_seed_means": [ + 0.007707856085617095, + 0.007097888254017259, + 0.010012049596601477 + ] + }, + "20": { + "mean_across_seeds": 0.005688730217193047, + "std_across_seeds": 0.0012039353173644645, + "per_seed_means": [ + 0.0049784621410071846, + 0.004703778505208902, + 0.007383950005363052 + ] + }, + "30": { + "mean_across_seeds": 0.004169839709583256, + "std_across_seeds": 0.001364214691961173, + "per_seed_means": [ + 0.0033304808987304566, + 0.003085115519352257, + 0.006093922710667054 + ] + }, + "50": { + "mean_across_seeds": 0.003013845277988973, + "std_across_seeds": 0.00137963251205373, + "per_seed_means": [ + 0.002109092113096267, + 0.001969176408407899, + 0.004963267312462752 + ] + }, + "100": { + "mean_across_seeds": 0.0018553994908062225, + "std_across_seeds": 0.0011837757725552231, + "per_seed_means": [ + 0.0010780306657155354, + 0.0009600430543650874, + 0.0035281247523380444 + ] + } + }, + "fit_power_law": { + "gamma": 0.6741618914822415, + "log_A": -3.179715570803609, + "R2": 0.999287, + "n_points": 3 + }, + "fit_exponential": { + "lambda": 0.011209420867769772, + "log_A": -5.185458562794302, + "R2": 0.983538, + "n_points": 3 + }, + "delta_R2_power_minus_exp": 0.0157, + "gamma_CI_95": { + "lo": 0.6355609753595243, + "hi": 1.4829672684297877 + }, + "decision": "UNCLEAR: \u03b3=0.674 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-2.8b_random.json b/data/e4_gamma/EleutherAI--pythia-2.8b_random.json new file mode 100644 index 0000000000000000000000000000000000000000..56c2de574e54c183ba2f5e49c2547baa7358134d --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-2.8b_random.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-2.8b", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.007388996658846736, + "std_across_seeds": 0.00018226253624295958, + "per_seed_means": [ + 0.007328578910479943, + 0.007202199265981714, + 0.007636211800078551 + ] + }, + "20": { + "mean_across_seeds": 0.0065620680425005655, + "std_across_seeds": 0.0013119660642064493, + "per_seed_means": [ + 0.008389089189780255, + 0.005368597246706486, + 0.005928517691014956 + ] + }, + "30": { + "mean_across_seeds": 0.004171221432948693, + "std_across_seeds": 5.492056749426419e-05, + "per_seed_means": [ + 0.004100738317550471, + 0.004178205231825511, + 0.0042347207494700945 + ] + }, + "50": { + "mean_across_seeds": 0.0028834379232850755, + "std_across_seeds": 0.00011670489556831891, + "per_seed_means": [ + 0.002764531443050752, + 0.0030420172734496494, + 0.0028437650533548247 + ] + }, + "100": { + "mean_across_seeds": 0.0016150823904253128, + "std_across_seeds": 1.6027428565874314e-05, + "per_seed_means": [ + 0.0015924673308230315, + 0.001627707876614295, + 0.001625071963838612 + ] + }, + "200": { + "mean_across_seeds": 0.0012179727724287658, + "std_across_seeds": 1.7310186855210638e-05, + "per_seed_means": [ + 0.001211881641841804, + 0.0012004845253735159, + 0.0012415521500709777 + ] + }, + "500": { + "mean_across_seeds": 0.0003523622222484037, + "std_across_seeds": 3.2832611113364645e-05, + "per_seed_means": [ + 0.00039570461075830583, + 0.00031626713990893525, + 0.0003451149160779702 + ] + }, + "1000": { + "mean_across_seeds": 0.00021088249445407807, + "std_across_seeds": 8.525966582865826e-06, + "per_seed_means": [ + 0.0002150495473082022, + 0.00019900024810340255, + 0.00021859768795062944 + ] + }, + "2000": { + "mean_across_seeds": 0.0009888755242556928, + "std_across_seeds": 0.0010801453503042908, + "per_seed_means": [ + 0.00020996567844122182, + 0.0002403297636192292, + 0.0025163311307066275 + ] + } + }, + "fit_power_law": { + "gamma": 0.5511942093739615, + "log_A": -3.841280530850879, + "R2": 0.644437, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0007053287474078895, + "log_A": -6.437797432208634, + "R2": 0.227415, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.417, + "gamma_CI_95": { + "lo": 0.23611731057176769, + "hi": 0.9185010077352719 + }, + "decision": "UNCLEAR: \u03b3=0.551 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-31m_mongo.json b/data/e4_gamma/EleutherAI--pythia-31m_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..055141f996d60388575c1ea10eaba464074528d1 --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-31m_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-31m", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.015075270254164932, + "std_across_seeds": 0.0016076363765655506, + "per_seed_means": [ + 0.015652700106923777, + 0.01288217271057268, + 0.016690937944998343 + ] + }, + "20": { + "mean_across_seeds": 0.007979718983923603, + "std_across_seeds": 0.0008011809172800115, + "per_seed_means": [ + 0.007325330031647657, + 0.007505871877074241, + 0.009107955043048909 + ] + }, + "30": { + "mean_across_seeds": 0.004943025353131816, + "std_across_seeds": 0.0008214856215946895, + "per_seed_means": [ + 0.004706252690715095, + 0.004076418298839902, + 0.0060464050698404515 + ] + }, + "50": { + "mean_across_seeds": 0.003909375242526746, + "std_across_seeds": 0.0004103009057058668, + "per_seed_means": [ + 0.004045853732774655, + 0.0033527197137785455, + 0.004329552281027039 + ] + }, + "100": { + "mean_across_seeds": 0.001335802334417369, + "std_across_seeds": 0.00014613574339847555, + "per_seed_means": [ + 0.0012400429498666198, + 0.0012250753167124156, + 0.0015422887366730719 + ] + }, + "200": { + "mean_across_seeds": 0.0006274800656319712, + "std_across_seeds": 8.133098864096775e-05, + "per_seed_means": [ + 0.000617688476195326, + 0.0005331277462513148, + 0.0007316239744492729 + ] + }, + "500": { + "mean_across_seeds": 0.0003229697247034993, + "std_across_seeds": 0.00010411438760957841, + "per_seed_means": [ + 0.0002789432165203228, + 0.0002233032413945087, + 0.00046666271619566637 + ] + }, + "1000": { + "mean_across_seeds": 0.00010229767169578944, + "std_across_seeds": 6.545994570499704e-05, + "per_seed_means": [ + 0.00014693857560094632, + 9.742619926242924e-06, + 0.00015021181956017908 + ] + }, + "2000": { + "mean_across_seeds": 2.207666448479115e-05, + "std_across_seeds": 9.890072727231223e-06, + "per_seed_means": [ + 3.0325915145184013e-05, + 8.170285452860545e-06, + 2.7733792856328893e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.2350013988825523, + "log_A": -0.8481173688844952, + "R2": 0.973742, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0025245756131658517, + "log_A": -6.1424841272489665, + "R2": 0.876899, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.0968, + "gamma_CI_95": { + "lo": 1.0142831942090689, + "hi": 1.3931415910718121 + }, + "decision": "ANOMALY: long-context training effect" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-31m_random.json b/data/e4_gamma/EleutherAI--pythia-31m_random.json new file mode 100644 index 0000000000000000000000000000000000000000..ba548b3b58c98ad1e1c2ce9aa1843d8767d5033d --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-31m_random.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-31m", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.01301000003495978, + "std_across_seeds": 0.00046688234450363445, + "per_seed_means": [ + 0.013054606715838114, + 0.01241719133220613, + 0.013558202056835096 + ] + }, + "20": { + "mean_across_seeds": 0.00693967725809974, + "std_across_seeds": 0.0003111505620949703, + "per_seed_means": [ + 0.006937875649115691, + 0.006559501201457654, + 0.007321654923725873 + ] + }, + "30": { + "mean_across_seeds": 0.004015894493398567, + "std_across_seeds": 0.00011863265749955042, + "per_seed_means": [ + 0.0040281298368548355, + 0.0038648689771071075, + 0.004154684666233758 + ] + }, + "50": { + "mean_across_seeds": 0.0036222275638202617, + "std_across_seeds": 0.0001889761810972585, + "per_seed_means": [ + 0.003356990779672439, + 0.003726471992752825, + 0.0037832199190355217 + ] + }, + "100": { + "mean_across_seeds": 0.0010882257377185548, + "std_across_seeds": 0.00012946031918837256, + "per_seed_means": [ + 0.0009092199558411569, + 0.0012110101386497262, + 0.0011444471186647812 + ] + }, + "200": { + "mean_across_seeds": 0.0005025619622918183, + "std_across_seeds": 6.674565621401741e-05, + "per_seed_means": [ + 0.0004423195596852262, + 0.0005956165699171834, + 0.00046974975727304504 + ] + }, + "500": { + "mean_across_seeds": 0.00019388230073268966, + "std_across_seeds": 9.190553129509277e-06, + "per_seed_means": [ + 0.0001877195072908459, + 0.00018705338045644262, + 0.00020687401445078043 + ] + }, + "1000": { + "mean_across_seeds": 2.1299731392648228e-05, + "std_across_seeds": 7.105662382177325e-06, + "per_seed_means": [ + 2.9809543584254546e-05, + 2.167332172462011e-05, + 1.2416328869070034e-05 + ] + }, + "2000": { + "mean_across_seeds": 7.186259580025636e-06, + "std_across_seeds": 8.041626579142191e-07, + "per_seed_means": [ + 6.1229779024548255e-06, + 7.368486014153556e-06, + 8.067314823468526e-06 + ] + } + }, + "fit_power_law": { + "gamma": 1.5398244746231446, + "log_A": 0.24504650180534032, + "R2": 0.964259, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.003151060152234357, + "log_A": -6.354208754917211, + "R2": 0.870219, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.094, + "gamma_CI_95": { + "lo": 1.1197173993826866, + "hi": 1.769418762310705 + }, + "decision": "ANOMALY: long-context training effect" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-410m_mongo.json b/data/e4_gamma/EleutherAI--pythia-410m_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..1830cde67d1d48f2ef12bb04264b490a9cdb6dd1 --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-410m_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-410m", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.010637531184022211, + "std_across_seeds": 0.0009754603695845838, + "per_seed_means": [ + 0.011129466105097284, + 0.009275416589807718, + 0.01150771085716163 + ] + }, + "20": { + "mean_across_seeds": 0.007524669199354119, + "std_across_seeds": 0.0003411354121354146, + "per_seed_means": [ + 0.007683635454935332, + 0.007050715095053116, + 0.007839657048073908 + ] + }, + "30": { + "mean_across_seeds": 0.004181321816156721, + "std_across_seeds": 0.0005739251464089627, + "per_seed_means": [ + 0.004148942732523816, + 0.0034951590199489148, + 0.004899863695997434 + ] + }, + "50": { + "mean_across_seeds": 0.0032434275430083897, + "std_across_seeds": 0.00038951675769930105, + "per_seed_means": [ + 0.0033627199286517377, + 0.0027180432780490567, + 0.003649519422324374 + ] + }, + "100": { + "mean_across_seeds": 0.001533225679288282, + "std_across_seeds": 0.00015696365531593024, + "per_seed_means": [ + 0.0016638820634883207, + 0.0013124849214606609, + 0.0016233100529158644 + ] + }, + "200": { + "mean_across_seeds": 0.000936404794475594, + "std_across_seeds": 0.00014957001750693787, + "per_seed_means": [ + 0.0009389797098507794, + 0.0007519457981591889, + 0.0011182888754168137 + ] + }, + "500": { + "mean_across_seeds": 0.0003803397886238397, + "std_across_seeds": 4.863341317573991e-05, + "per_seed_means": [ + 0.00040086741908453407, + 0.0003132272653480565, + 0.00042692468143892863 + ] + }, + "1000": { + "mean_across_seeds": 0.0001626184749986553, + "std_across_seeds": 1.772572747902181e-05, + "per_seed_means": [ + 0.00017353555046308126, + 0.00013761728597576925, + 0.00017670258855711533 + ] + }, + "2000": { + "mean_across_seeds": 5.13333430424407e-05, + "std_across_seeds": 3.9984125574520355e-06, + "per_seed_means": [ + 5.496963710811542e-05, + 4.5764998872073194e-05, + 5.3265393147133486e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.0218530106365162, + "log_A": -1.7669627940483377, + "R2": 0.981594, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0020820749808360484, + "log_A": -6.151337865679417, + "R2": 0.878237, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.1034, + "gamma_CI_95": { + "lo": 0.846278741222938, + "hi": 1.1560602030295848 + }, + "decision": "CONFIRMED: \u03b3 law holds (\u03b3\u00d7ln(\u03b8) = C)" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-410m_random.json b/data/e4_gamma/EleutherAI--pythia-410m_random.json new file mode 100644 index 0000000000000000000000000000000000000000..66597a353362f59f2604bc889fa0fbe9fe2df40b --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-410m_random.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-410m", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.009754525538430446, + "std_across_seeds": 0.0003722593944128235, + "per_seed_means": [ + 0.009823903981596232, + 0.009267889905410508, + 0.010171782728284597 + ] + }, + "20": { + "mean_across_seeds": 0.009123972739713887, + "std_across_seeds": 0.0007766048034381784, + "per_seed_means": [ + 0.010081037391598026, + 0.008178864621246855, + 0.009112016206296782 + ] + }, + "30": { + "mean_across_seeds": 0.0038759738756602422, + "std_across_seeds": 8.204626140183602e-05, + "per_seed_means": [ + 0.0038872477606249354, + 0.003770326642940442, + 0.0039703472234153495 + ] + }, + "50": { + "mean_across_seeds": 0.0032870457074346223, + "std_across_seeds": 7.352391975717852e-05, + "per_seed_means": [ + 0.0032759768589554973, + 0.0033821164964077375, + 0.0032030437669406334 + ] + }, + "100": { + "mean_across_seeds": 0.0016699490632163362, + "std_across_seeds": 4.519511018857947e-05, + "per_seed_means": [ + 0.0017094237510658179, + 0.001693745714534695, + 0.0016066777240484954 + ] + }, + "200": { + "mean_across_seeds": 0.001006049710429377, + "std_across_seeds": 1.7124807153708726e-05, + "per_seed_means": [ + 0.0010300154409681758, + 0.0009970870024214188, + 0.0009910466878985365 + ] + }, + "500": { + "mean_across_seeds": 0.0004026397552903897, + "std_across_seeds": 2.1794816158143854e-05, + "per_seed_means": [ + 0.0004279522445479718, + 0.0003747526396182366, + 0.00040521438170496065 + ] + }, + "1000": { + "mean_across_seeds": 0.0001901201162480801, + "std_across_seeds": 1.22550448532374e-05, + "per_seed_means": [ + 0.00018012737991133083, + 0.00020737976708915085, + 0.0001828532017437586 + ] + }, + "2000": { + "mean_across_seeds": 7.827874543484844e-05, + "std_across_seeds": 3.882609187098134e-06, + "per_seed_means": [ + 7.300645292464954e-05, + 8.224311603650373e-05, + 7.958666734339203e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.9362347719713631, + "log_A": -2.126749270781839, + "R2": 0.987529, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.001877382921682511, + "log_A": -6.160531693807699, + "R2": 0.855756, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.1318, + "gamma_CI_95": { + "lo": 0.7880672419289397, + "hi": 1.0390619514918187 + }, + "decision": "CONFIRMED: \u03b3 law holds (\u03b3\u00d7ln(\u03b8) = C)" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-70m_layerwise.json b/data/e4_gamma/EleutherAI--pythia-70m_layerwise.json new file mode 100644 index 0000000000000000000000000000000000000000..6b3393afd18a7248375c4c859429e3b799cca0a7 --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-70m_layerwise.json @@ -0,0 +1,143 @@ +{ + "model": "EleutherAI/pythia-70m", + "theta": 10000, + "d_head": 64, + "n_layers": 6, + "L_crit_known": 4, + "z": 0.282842712474619, + "gamma_pos_pred": 0.7522013138014093, + "gamma_pade": 0.7522013138014093, + "beta_gamma": 0.2, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "n_prompts": 150, + "seeds": [ + 42, + 123, + 7 + ], + "layer_gamma": { + "0": { + "gamma": 1.3468013543313044, + "R2": 0.981851, + "n_points": 7, + "type": "pos" + }, + "1": { + "gamma": 1.410356134563726, + "R2": 0.886404, + "n_points": 7, + "type": "pos" + }, + "2": { + "gamma": 1.1152541530423556, + "R2": 0.949761, + "n_points": 7, + "type": "pos" + }, + "3": { + "gamma": 1.1030121418119894, + "R2": 0.992778, + "n_points": 7, + "type": "pos" + }, + "4": { + "gamma": NaN, + "R2": 0.0, + "n_points": 0, + "type": "sem" + }, + "5": { + "gamma": NaN, + "R2": 0.0, + "n_points": 0, + "type": "sem" + } + }, + "per_layer_agg": { + "0": { + "10": 0.015644904689656364, + "20": 0.013133970548709234, + "30": 0.005567839278115167, + "50": 0.003556306262811025, + "100": 0.0019510852628284031, + "200": 0.0007665379345417021, + "500": 0.0002498780190944672, + "1000": 5.03444837199317e-05, + "2000": 2.262747950024075e-05 + }, + "1": { + "10": 0.008926776730351978, + "20": 0.01036835898955663, + "30": 0.0026452516516049704, + "50": 0.0025659463637404973, + "100": 0.0008467986186345418, + "200": 0.0005270672672324711, + "500": 0.00018555617994732326, + "1000": 8.427821927600437e-05, + "2000": 3.328240580028958e-06 + }, + "2": { + "10": 0.016187396844228107, + "20": 0.010158376197020213, + "30": 0.0047512871026992795, + "50": 0.0032849965658452774, + "100": 0.0013126978443728553, + "200": 0.0007603488696946038, + "500": 0.00015906307432386608, + "1000": 5.299210548400879e-05, + "2000": 8.578023976749844e-05 + }, + "3": { + "10": 0.01852987120057353, + "20": 0.014939713157816894, + "30": 0.007211493629470674, + "50": 0.005900978506348717, + "100": 0.0021263419639500273, + "200": 0.001293775682166193, + "500": 0.00036153811212797233, + "1000": 0.00019700892104083324, + "2000": 7.671554883321127e-05 + }, + "4": { + "10": NaN, + "20": NaN, + "30": NaN, + "50": NaN, + "100": NaN, + "200": NaN, + "500": NaN, + "1000": NaN, + "2000": NaN + }, + "5": { + "10": NaN, + "20": NaN, + "30": NaN, + "50": NaN, + "100": NaN, + "200": NaN, + "500": NaN, + "1000": NaN, + "2000": NaN + } + } +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-70m_mongo.json b/data/e4_gamma/EleutherAI--pythia-70m_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..6436b17d7d58d5fa9fb77ea8d55539d9e2464633 --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-70m_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-70m", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.015389821239643628, + "std_across_seeds": 0.0009669444035724801, + "per_seed_means": [ + 0.015509476736187935, + 0.014150275668750207, + 0.016509711313992738 + ] + }, + "20": { + "mean_across_seeds": 0.011693157334811985, + "std_across_seeds": 0.0004602414734324704, + "per_seed_means": [ + 0.011049262892144422, + 0.01193274388089776, + 0.012097465231393774 + ] + }, + "30": { + "mean_across_seeds": 0.007082567571972807, + "std_across_seeds": 0.0005349392513652303, + "per_seed_means": [ + 0.006596428078288833, + 0.006823649969883263, + 0.007827624667746325 + ] + }, + "50": { + "mean_across_seeds": 0.0054180885271893605, + "std_across_seeds": 0.0004311082027556252, + "per_seed_means": [ + 0.005146260828090211, + 0.005081388686473171, + 0.006026616067004701 + ] + }, + "100": { + "mean_across_seeds": 0.002821484198017667, + "std_across_seeds": 0.0001332760135493066, + "per_seed_means": [ + 0.002738000506845613, + 0.002716881933156401, + 0.003009570154050986 + ] + }, + "200": { + "mean_across_seeds": 0.0018563934957556841, + "std_across_seeds": 0.0001484262285601927, + "per_seed_means": [ + 0.0016465914276583742, + 0.0019555641586581868, + 0.0019670249009504912 + ] + }, + "500": { + "mean_across_seeds": 0.0007652510016729745, + "std_across_seeds": 0.00010428199338802599, + "per_seed_means": [ + 0.0007198545018521448, + 0.000666431888336471, + 0.0009094666148303077 + ] + }, + "1000": { + "mean_across_seeds": 0.0004260873921642391, + "std_across_seeds": 8.04926254607836e-05, + "per_seed_means": [ + 0.0003130952876138811, + 0.0004706175611742462, + 0.0004945493277045899 + ] + }, + "2000": { + "mean_across_seeds": 0.00039341128339098454, + "std_across_seeds": 0.0003178948846095019, + "per_seed_means": [ + 0.00016619697392646535, + 0.00017106311420017544, + 0.0008429737620463129 + ] + } + }, + "fit_power_law": { + "gamma": 0.7476017873166874, + "log_A": -2.391511197086578, + "R2": 0.984269, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0013585785276598904, + "log_A": -5.69046941405711, + "R2": 0.700497, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2838, + "gamma_CI_95": { + "lo": 0.6824154281049325, + "hi": 0.852609361439985 + }, + "decision": "UNCLEAR: \u03b3=0.748 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/EleutherAI--pythia-70m_random.json b/data/e4_gamma/EleutherAI--pythia-70m_random.json new file mode 100644 index 0000000000000000000000000000000000000000..d5fe29e93a00f9b42260e01dd548af7fa115d83e --- /dev/null +++ b/data/e4_gamma/EleutherAI--pythia-70m_random.json @@ -0,0 +1,134 @@ +{ + "model": "EleutherAI/pythia-70m", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.014303823270731502, + "std_across_seeds": 0.0005389411970747021, + "per_seed_means": [ + 0.013846093355678022, + 0.01400491089404871, + 0.015060465562467774 + ] + }, + "20": { + "mean_across_seeds": 0.011754385376763014, + "std_across_seeds": 0.0003750825584104145, + "per_seed_means": [ + 0.012279952564276754, + 0.011553768791879217, + 0.011429434774133067 + ] + }, + "30": { + "mean_across_seeds": 0.004743960113911372, + "std_across_seeds": 0.0004785557585293483, + "per_seed_means": [ + 0.004255784423633789, + 0.004582108563821142, + 0.005393987354279185 + ] + }, + "50": { + "mean_across_seeds": 0.0035309766319632113, + "std_across_seeds": 0.00030331972886044056, + "per_seed_means": [ + 0.003315780726649488, + 0.003317214461664359, + 0.003959934707575788 + ] + }, + "100": { + "mean_across_seeds": 0.0014786803761186699, + "std_across_seeds": 4.0198997401330876e-05, + "per_seed_means": [ + 0.0015339826285101783, + 0.0014396193967938112, + 0.00146243910305202 + ] + }, + "200": { + "mean_across_seeds": 0.0007715668188918952, + "std_across_seeds": 6.92475008380853e-05, + "per_seed_means": [ + 0.0008646439060976263, + 0.0007513971288184015, + 0.0006986594217596576 + ] + }, + "500": { + "mean_across_seeds": 0.0002215023043552517, + "std_across_seeds": 2.714460980421038e-05, + "per_seed_means": [ + 0.00024024882055527995, + 0.00024114061049961797, + 0.0001831174820108572 + ] + }, + "1000": { + "mean_across_seeds": 8.175638578702798e-05, + "std_across_seeds": 1.2207321420922821e-05, + "per_seed_means": [ + 8.717483436763965e-05, + 9.324252537529295e-05, + 6.485179761815137e-05 + ] + }, + "2000": { + "mean_across_seeds": 4.288268759245915e-05, + "std_across_seeds": 9.477341124989812e-06, + "per_seed_means": [ + 3.2012329573566e-05, + 5.510804182828603e-05, + 4.152769137552544e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.1705141984482668, + "log_A": -1.1650521267460994, + "R2": 0.994076, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.00226006648331684, + "log_A": -6.2565121323123964, + "R2": 0.79868, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.1954, + "gamma_CI_95": { + "lo": 1.065560635977702, + "hi": 1.268916819874023 + }, + "decision": "ANOMALY: long-context training effect" +} \ No newline at end of file diff --git a/data/e4_gamma/HuggingFaceTB--SmolLM2-135M_mongo.json b/data/e4_gamma/HuggingFaceTB--SmolLM2-135M_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..167c8e26c88a2442b4e6ad1bfc3a6eceab6dfc62 --- /dev/null +++ b/data/e4_gamma/HuggingFaceTB--SmolLM2-135M_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "HuggingFaceTB/SmolLM2-135M", + "theta": 100000, + "gamma_pred": 0.799996493334704, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.008304463168606162, + "std_across_seeds": 0.0005410428815580668, + "per_seed_means": [ + 0.00755697955765451, + 0.008819775762191662, + 0.008536634185972313 + ] + }, + "20": { + "mean_across_seeds": 0.005037364546959806, + "std_across_seeds": 0.0004860118497491869, + "per_seed_means": [ + 0.004630992590682581, + 0.00572061057551764, + 0.004760490474679197 + ] + }, + "30": { + "mean_across_seeds": 0.003402677008933905, + "std_across_seeds": 0.0005495050355643191, + "per_seed_means": [ + 0.0028194241718544314, + 0.003249561061384156, + 0.004139045793563128 + ] + }, + "50": { + "mean_across_seeds": 0.0022955730223071036, + "std_across_seeds": 0.00032195150800856534, + "per_seed_means": [ + 0.0018683485605288297, + 0.0023728452197974547, + 0.0026455252865950265 + ] + }, + "100": { + "mean_across_seeds": 0.0012311791702975623, + "std_across_seeds": 0.00021720799356486474, + "per_seed_means": [ + 0.0009818656951150237, + 0.0012004297882473717, + 0.0015112420275302915 + ] + }, + "200": { + "mean_across_seeds": 0.0006159267034672667, + "std_across_seeds": 0.00016324775670508771, + "per_seed_means": [ + 0.00038703462055612663, + 0.0007042756974018024, + 0.0007564697924438709 + ] + }, + "500": { + "mean_across_seeds": 0.00033591168693318545, + "std_across_seeds": 0.00011815002481030313, + "per_seed_means": [ + 0.0001744628540230527, + 0.00045391677274892573, + 0.00037935543402757804 + ] + }, + "1000": { + "mean_across_seeds": 0.0002568992384011734, + "std_across_seeds": 0.00010632737830411309, + "per_seed_means": [ + 0.00010659140847565141, + 0.00032832029080964275, + 0.00033578601591822613 + ] + }, + "2000": { + "mean_across_seeds": 0.00014454515663096066, + "std_across_seeds": 6.508330086377912e-05, + "per_seed_means": [ + 5.251373916507873e-05, + 0.00018937018219730817, + 0.0001917515485304951 + ] + } + }, + "fit_power_law": { + "gamma": 0.7479961324735662, + "log_A": -3.2257978430772267, + "R2": 0.988937, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0013723290915239371, + "log_A": -6.519271668207533, + "R2": 0.717382, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2716, + "gamma_CI_95": { + "lo": 0.6597674707909884, + "hi": 0.8631975695160649 + }, + "decision": "CONFIRMED: \u03b3 law holds (\u03b3\u00d7ln(\u03b8) = C)" +} \ No newline at end of file diff --git a/data/e4_gamma/HuggingFaceTB--SmolLM2-135M_random.json b/data/e4_gamma/HuggingFaceTB--SmolLM2-135M_random.json new file mode 100644 index 0000000000000000000000000000000000000000..8643d9b1c8e1f8da486e521bc7dc3846d1b53379 --- /dev/null +++ b/data/e4_gamma/HuggingFaceTB--SmolLM2-135M_random.json @@ -0,0 +1,134 @@ +{ + "model": "HuggingFaceTB/SmolLM2-135M", + "theta": 100000, + "gamma_pred": 0.799996493334704, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.007505987734119924, + "std_across_seeds": 6.222712128447283e-05, + "per_seed_means": [ + 0.0074217521399259564, + 0.007526046622854968, + 0.007570164439578851 + ] + }, + "20": { + "mean_across_seeds": 0.005387026191326894, + "std_across_seeds": 7.928170861411101e-05, + "per_seed_means": [ + 0.005308992751718809, + 0.005356318663495283, + 0.005495767158766588 + ] + }, + "30": { + "mean_across_seeds": 0.0033108791688250174, + "std_across_seeds": 9.030220201566211e-06, + "per_seed_means": [ + 0.0033224673941731455, + 0.003300437251261125, + 0.003309732861040781 + ] + }, + "50": { + "mean_across_seeds": 0.0020586437733274786, + "std_across_seeds": 3.9822153653400244e-05, + "per_seed_means": [ + 0.0020233509328681978, + 0.0020382833857244503, + 0.0021142970013897865 + ] + }, + "100": { + "mean_across_seeds": 0.0016422379348013138, + "std_across_seeds": 0.0005776106795492194, + "per_seed_means": [ + 0.002451185053214431, + 0.0011395067275346566, + 0.0013360220236548533 + ] + }, + "200": { + "mean_across_seeds": 0.0006042598345407491, + "std_across_seeds": 2.0220872080823326e-05, + "per_seed_means": [ + 0.0006080910796299576, + 0.0005778020712508199, + 0.0006268863527414699 + ] + }, + "500": { + "mean_across_seeds": 0.0006011281119895608, + "std_across_seeds": 0.0005208506291629337, + "per_seed_means": [ + 0.00022182018486394857, + 0.00024395286166206158, + 0.0013376112894426721 + ] + }, + "1000": { + "mean_across_seeds": 0.00014980076856671882, + "std_across_seeds": 9.588413659831112e-06, + "per_seed_means": [ + 0.0001631340318514655, + 0.00014527296426725418, + 0.00014099530958143684 + ] + }, + "2000": { + "mean_across_seeds": 0.0003835183502128024, + "std_across_seeds": 0.00045868510780794716, + "per_seed_means": [ + 0.0010321856801844358, + 6.251086288102669e-05, + 5.5858507572944896e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.62665506839462, + "log_A": -3.719029155874015, + "R2": 0.828515, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0010285378547186114, + "log_A": -6.545393439522604, + "R2": 0.481003, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.3475, + "gamma_CI_95": { + "lo": 0.43228026562247185, + "hi": 0.9156890714410367 + }, + "decision": "UNCLEAR: \u03b3=0.627 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/HuggingFaceTB--SmolLM2-360M_mongo.json b/data/e4_gamma/HuggingFaceTB--SmolLM2-360M_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..3a7f8a735923cbfe0a1b5362d985fad63123b190 --- /dev/null +++ b/data/e4_gamma/HuggingFaceTB--SmolLM2-360M_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "HuggingFaceTB/SmolLM2-360M", + "theta": 100000, + "gamma_pred": 0.799996493334704, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.007966959143264426, + "std_across_seeds": 0.0005601652428789886, + "per_seed_means": [ + 0.007183899756831427, + 0.008462369282109042, + 0.008254608390852809 + ] + }, + "20": { + "mean_across_seeds": 0.004732916150759491, + "std_across_seeds": 0.00034992389259481266, + "per_seed_means": [ + 0.004537673588298882, + 0.005224339859948183, + 0.0044367350040314096 + ] + }, + "30": { + "mean_across_seeds": 0.0030590847538810016, + "std_across_seeds": 0.00036923530136224845, + "per_seed_means": [ + 0.0027501304377801714, + 0.002848990266599382, + 0.0035781335572634514 + ] + }, + "50": { + "mean_across_seeds": 0.002027262340480876, + "std_across_seeds": 0.00015466362617720856, + "per_seed_means": [ + 0.001853673771644632, + 0.001998811180431706, + 0.0022293020693662886 + ] + }, + "100": { + "mean_across_seeds": 0.0010873301881171452, + "std_across_seeds": 8.610188701535176e-05, + "per_seed_means": [ + 0.0010064105623556923, + 0.0010489908065452861, + 0.0012065891954504574 + ] + }, + "200": { + "mean_across_seeds": 0.0004853193103271122, + "std_across_seeds": 4.3515661719948895e-05, + "per_seed_means": [ + 0.00043014322674328774, + 0.0005365107220616968, + 0.0004893039821763523 + ] + }, + "500": { + "mean_across_seeds": 0.0002133857070940495, + "std_across_seeds": 2.844458225923954e-05, + "per_seed_means": [ + 0.00018429137151542818, + 0.0002519907450672084, + 0.00020387500469951192 + ] + }, + "1000": { + "mean_across_seeds": 0.00011797002824298236, + "std_across_seeds": 8.847258236115946e-06, + "per_seed_means": [ + 0.00011550892093509902, + 0.00012982452563543726, + 0.00010857663815841078 + ] + }, + "2000": { + "mean_across_seeds": 5.1520356169526674e-05, + "std_across_seeds": 4.1715296081625e-06, + "per_seed_means": [ + 4.59346058111502e-05, + 5.595701771198947e-05, + 5.266944498544035e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.9691725803316233, + "log_A": -2.4351546306959655, + "R2": 0.998139, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.001865324074347278, + "log_A": -6.654143898451364, + "R2": 0.796822, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2013, + "gamma_CI_95": { + "lo": 0.9317078799826332, + "hi": 1.0044633591334093 + }, + "decision": "REFUTED: C not constant across \u03b8" +} \ No newline at end of file diff --git a/data/e4_gamma/HuggingFaceTB--SmolLM2-360M_random.json b/data/e4_gamma/HuggingFaceTB--SmolLM2-360M_random.json new file mode 100644 index 0000000000000000000000000000000000000000..35ed4d4cf44d7940dcaf3851f7d4ce9dd9dd3e70 --- /dev/null +++ b/data/e4_gamma/HuggingFaceTB--SmolLM2-360M_random.json @@ -0,0 +1,134 @@ +{ + "model": "HuggingFaceTB/SmolLM2-360M", + "theta": 100000, + "gamma_pred": 0.799996493334704, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.0075793186652784545, + "std_across_seeds": 9.488817720498863e-05, + "per_seed_means": [ + 0.007506389223660032, + 0.007518230212541918, + 0.007713336559633414 + ] + }, + "20": { + "mean_across_seeds": 0.005817425185814499, + "std_across_seeds": 6.958758292443584e-05, + "per_seed_means": [ + 0.005819998332299292, + 0.005730940715099374, + 0.005901336510044833 + ] + }, + "30": { + "mean_across_seeds": 0.003471352700402753, + "std_across_seeds": 0.00013065052503733263, + "per_seed_means": [ + 0.0036349724791944027, + 0.0034638783040766914, + 0.0033152073179371654 + ] + }, + "50": { + "mean_across_seeds": 0.002386403470429488, + "std_across_seeds": 8.121991095048092e-05, + "per_seed_means": [ + 0.002459626552493622, + 0.002273151234646017, + 0.002426432624148826 + ] + }, + "100": { + "mean_across_seeds": 0.0015280379799918996, + "std_across_seeds": 0.00010151077379917431, + "per_seed_means": [ + 0.0016437112788359324, + 0.0013965710260284445, + 0.001543831635111322 + ] + }, + "200": { + "mean_across_seeds": 0.0007104039443462778, + "std_across_seeds": 4.436987859585186e-05, + "per_seed_means": [ + 0.0007541682361625135, + 0.0006495789651914189, + 0.0007274646316849006 + ] + }, + "500": { + "mean_across_seeds": 0.0003033048584701545, + "std_across_seeds": 3.6902252588477966e-05, + "per_seed_means": [ + 0.0002615151315209611, + 0.00029712799480572966, + 0.0003512714490837728 + ] + }, + "1000": { + "mean_across_seeds": 0.0001780933051163124, + "std_across_seeds": 1.2855955644831184e-05, + "per_seed_means": [ + 0.00017675424072270592, + 0.00019446533944574184, + 0.00016306033518048935 + ] + }, + "2000": { + "mean_across_seeds": 6.960754264582647e-05, + "std_across_seeds": 7.328966477321964e-06, + "per_seed_means": [ + 7.937209645509331e-05, + 6.773537223731789e-05, + 6.171515924506821e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.9197098362636822, + "log_A": -2.406781712821382, + "R2": 0.992716, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0018141314423132892, + "log_A": -6.3860584376391865, + "R2": 0.832386, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.1603, + "gamma_CI_95": { + "lo": 0.8428163690203526, + "hi": 1.0093623120299413 + }, + "decision": "REFUTED: C not constant across \u03b8" +} \ No newline at end of file diff --git a/data/e4_gamma/Qwen--Qwen2.5-0.5B_mongo.json b/data/e4_gamma/Qwen--Qwen2.5-0.5B_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..e221f5270f7147e63c6853e9ce9010bf9bb26fc1 --- /dev/null +++ b/data/e4_gamma/Qwen--Qwen2.5-0.5B_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "Qwen/Qwen2.5-0.5B", + "theta": 1000000, + "gamma_pred": 0.6666637444455867, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.010570011463585412, + "std_across_seeds": 0.0006879723804669801, + "per_seed_means": [ + 0.009635474333384384, + 0.011271671279876803, + 0.010802888777495051 + ] + }, + "20": { + "mean_across_seeds": 0.00707240852061659, + "std_across_seeds": 0.000389520100149214, + "per_seed_means": [ + 0.0068374204235927515, + 0.007621381778735667, + 0.006758423359521354 + ] + }, + "30": { + "mean_across_seeds": 0.004906498004501271, + "std_across_seeds": 0.00017033846135921013, + "per_seed_means": [ + 0.004929499562131241, + 0.005102665189964076, + 0.004687329261408498 + ] + }, + "50": { + "mean_across_seeds": 0.0027538882062395314, + "std_across_seeds": 9.39742235656474e-05, + "per_seed_means": [ + 0.0027291350761273255, + 0.00287934524686231, + 0.002653184295728958 + ] + }, + "100": { + "mean_across_seeds": 0.001320942670281511, + "std_across_seeds": 0.00010363790555285399, + "per_seed_means": [ + 0.0012310828099725767, + 0.00146614785825174, + 0.0012655973426202157 + ] + }, + "200": { + "mean_across_seeds": 0.000577603733767723, + "std_across_seeds": 4.0297786585279594e-05, + "per_seed_means": [ + 0.0006196609819987013, + 0.0005898806393573371, + 0.0005232695799471306 + ] + }, + "500": { + "mean_across_seeds": 0.00024497756287019003, + "std_across_seeds": 1.2471281191936438e-05, + "per_seed_means": [ + 0.0002595857172430745, + 0.00022911482625507536, + 0.00024623214511242016 + ] + }, + "1000": { + "mean_across_seeds": 0.00011319100446043093, + "std_across_seeds": 1.2651024110541512e-05, + "per_seed_means": [ + 0.00012925400123549479, + 0.00011198282766296567, + 9.833618448283232e-05 + ] + }, + "2000": { + "mean_across_seeds": 7.040919211400856e-05, + "std_across_seeds": 5.101730843717623e-06, + "per_seed_means": [ + 7.729242793478382e-05, + 6.509479955639108e-05, + 6.88403488508508e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.0283740139174087, + "log_A": -1.8919214095663512, + "R2": 0.996775, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0019054864999929597, + "log_A": -6.409520766789601, + "R2": 0.737515, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2593, + "gamma_CI_95": { + "lo": 0.9717583644853511, + "hi": 1.0942794404578478 + }, + "decision": "REFUTED: C not constant across \u03b8" +} \ No newline at end of file diff --git a/data/e4_gamma/Qwen--Qwen2.5-0.5B_random.json b/data/e4_gamma/Qwen--Qwen2.5-0.5B_random.json new file mode 100644 index 0000000000000000000000000000000000000000..48306034352e92e9536d0a06f8a9928f50aa97b9 --- /dev/null +++ b/data/e4_gamma/Qwen--Qwen2.5-0.5B_random.json @@ -0,0 +1,134 @@ +{ + "model": "Qwen/Qwen2.5-0.5B", + "theta": 1000000, + "gamma_pred": 0.6666637444455867, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.010635459372877248, + "std_across_seeds": 0.00025287936403276716, + "per_seed_means": [ + 0.01028363746435692, + 0.010755800842307508, + 0.010866939811967314 + ] + }, + "20": { + "mean_across_seeds": 0.008148884469539755, + "std_across_seeds": 0.0004449303658328976, + "per_seed_means": [ + 0.008704437670918802, + 0.008126969210182628, + 0.007615246527517835 + ] + }, + "30": { + "mean_across_seeds": 0.005823801488149912, + "std_across_seeds": 0.0002978411235393803, + "per_seed_means": [ + 0.005403217209192614, + 0.006014202661657085, + 0.006053984593600035 + ] + }, + "50": { + "mean_across_seeds": 0.004366586864408519, + "std_across_seeds": 0.0003111265487346291, + "per_seed_means": [ + 0.004062428347145518, + 0.00424332035627837, + 0.0047940118898016715 + ] + }, + "100": { + "mean_across_seeds": 0.0018307541850824942, + "std_across_seeds": 0.0001194163000240852, + "per_seed_means": [ + 0.001711244311882183, + 0.0017871726725328092, + 0.001993845570832491 + ] + }, + "200": { + "mean_across_seeds": 0.0009602205806974476, + "std_across_seeds": 2.195872155884662e-05, + "per_seed_means": [ + 0.000986584196604478, + 0.0009612513726460748, + 0.0009328261728417905 + ] + }, + "500": { + "mean_across_seeds": 0.0004414605432733272, + "std_across_seeds": 1.0677396043156196e-05, + "per_seed_means": [ + 0.00044303275436201756, + 0.0004536804476326021, + 0.00042766842782536213 + ] + }, + "1000": { + "mean_across_seeds": 0.00022347296873502073, + "std_across_seeds": 7.233574415262604e-06, + "per_seed_means": [ + 0.00021570391957842123, + 0.00023312102561855378, + 0.00022159396100808712 + ] + }, + "2000": { + "mean_across_seeds": 0.00013890501313047328, + "std_across_seeds": 5.059921830410724e-06, + "per_seed_means": [ + 0.00013592033639724831, + 0.0001347650347452145, + 0.00014602966824895703 + ] + } + }, + "fit_power_law": { + "gamma": 0.9194920254637005, + "log_A": -1.9941156229372259, + "R2": 0.995818, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0017142700743956925, + "log_A": -6.027563543680847, + "R2": 0.745944, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2499, + "gamma_CI_95": { + "lo": 0.8634080075020839, + "hi": 0.9885081580114671 + }, + "decision": "REFUTED: C not constant across \u03b8" +} \ No newline at end of file diff --git a/data/e4_gamma/Qwen--Qwen2.5-3B_mongo.json b/data/e4_gamma/Qwen--Qwen2.5-3B_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..86411cbd095b8c008b102b6290d968502cd0b98d --- /dev/null +++ b/data/e4_gamma/Qwen--Qwen2.5-3B_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "Qwen/Qwen2.5-3B", + "theta": 1000000, + "gamma_pred": 0.6666637444455867, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.01112439402223875, + "std_across_seeds": 0.000838491547598414, + "per_seed_means": [ + 0.01075855381709213, + 0.01228415741895636, + 0.010330470830667763 + ] + }, + "20": { + "mean_across_seeds": 0.006511005685897544, + "std_across_seeds": 0.0007180136286557984, + "per_seed_means": [ + 0.006058958645056312, + 0.007524465210735798, + 0.005949593201900522 + ] + }, + "30": { + "mean_across_seeds": 0.0043632738612359384, + "std_across_seeds": 0.00034911261508580775, + "per_seed_means": [ + 0.004442217379886036, + 0.0047458748061520355, + 0.003901729397669745 + ] + }, + "50": { + "mean_across_seeds": 0.00307781646283628, + "std_across_seeds": 0.00041331004232130885, + "per_seed_means": [ + 0.0030109512369381262, + 0.0036141253705136477, + 0.002608372781057066 + ] + }, + "100": { + "mean_across_seeds": 0.0016049638397978721, + "std_across_seeds": 0.0004841336652166786, + "per_seed_means": [ + 0.0012929666247024822, + 0.0022887610945811804, + 0.001233163800109954 + ] + }, + "200": { + "mean_across_seeds": 0.0008450243635565535, + "std_across_seeds": 0.0002811253249757243, + "per_seed_means": [ + 0.0008146637216365585, + 0.0012035060769994743, + 0.0005169032920336273 + ] + }, + "500": { + "mean_across_seeds": 0.00045879156515285204, + "std_across_seeds": 0.00016521171054225374, + "per_seed_means": [ + 0.0005418163688470182, + 0.0006064154248451814, + 0.00022814290176635646 + ] + }, + "1000": { + "mean_across_seeds": 0.0002865213390759891, + "std_across_seeds": 9.734556435503643e-05, + "per_seed_means": [ + 0.0002817838863484212, + 0.0004080429326131707, + 0.00016973719826637534 + ] + }, + "2000": { + "mean_across_seeds": 0.00017661211525086802, + "std_across_seeds": 0.00010037099387547912, + "per_seed_means": [ + 0.00016309323025780033, + 0.00030574162936924644, + 6.100148612555737e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.7720333740995954, + "log_A": -2.8463010634461146, + "R2": 0.995841, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0014323846371857994, + "log_A": -6.236768717117085, + "R2": 0.738756, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2571, + "gamma_CI_95": { + "lo": 0.7129891077884591, + "hi": 0.8507556651066346 + }, + "decision": "UNCLEAR: \u03b3=0.772 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/Qwen--Qwen2.5-3B_random.json b/data/e4_gamma/Qwen--Qwen2.5-3B_random.json new file mode 100644 index 0000000000000000000000000000000000000000..6bfb12c341702ad2752048b56603383742a98dad --- /dev/null +++ b/data/e4_gamma/Qwen--Qwen2.5-3B_random.json @@ -0,0 +1,134 @@ +{ + "model": "Qwen/Qwen2.5-3B", + "theta": 1000000, + "gamma_pred": 0.6666637444455867, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.011255048217458858, + "std_across_seeds": 0.00029170327713132915, + "per_seed_means": [ + 0.011365672556372981, + 0.010855558921272556, + 0.011543913174731036 + ] + }, + "20": { + "mean_across_seeds": 0.008040837371307943, + "std_across_seeds": 5.735484570626992e-05, + "per_seed_means": [ + 0.008052886674801508, + 0.00796534705751886, + 0.00810427838160346 + ] + }, + "30": { + "mean_across_seeds": 0.005607237855696843, + "std_across_seeds": 8.863202668815811e-05, + "per_seed_means": [ + 0.005691748578877499, + 0.005645150489484271, + 0.0054848144987287625 + ] + }, + "50": { + "mean_across_seeds": 0.004305336697854929, + "std_across_seeds": 0.00013922616269980767, + "per_seed_means": [ + 0.004180670159403235, + 0.004235686721124996, + 0.004499653213036557 + ] + }, + "100": { + "mean_across_seeds": 0.0017492879890940256, + "std_across_seeds": 0.00013272472868903432, + "per_seed_means": [ + 0.0016116862614095832, + 0.0017075307061895728, + 0.0019286469996829207 + ] + }, + "200": { + "mean_across_seeds": 0.0007549225819174252, + "std_across_seeds": 7.111260830671966e-05, + "per_seed_means": [ + 0.0007408848872485881, + 0.0006756992589604731, + 0.0008481835995432144 + ] + }, + "500": { + "mean_across_seeds": 0.0003144526818747787, + "std_across_seeds": 5.0788319188644444e-05, + "per_seed_means": [ + 0.00028346924572057714, + 0.00038606213388751105, + 0.00027382666601624803 + ] + }, + "1000": { + "mean_across_seeds": 0.0002920797811627078, + "std_across_seeds": 3.889070069088199e-05, + "per_seed_means": [ + 0.00030768415017519147, + 0.0003299515241330179, + 0.0002386036691799139 + ] + }, + "2000": { + "mean_across_seeds": 8.988117967318329e-05, + "std_across_seeds": 5.94207520177485e-06, + "per_seed_means": [ + 9.827569079789101e-05, + 8.601787588607598e-05, + 8.534997233558291e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.9643628833140916, + "log_A": -1.871616966615359, + "R2": 0.97891, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0018280361467505031, + "log_A": -6.085205860136052, + "R2": 0.758046, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2209, + "gamma_CI_95": { + "lo": 0.8066661732202158, + "hi": 1.1104644097863652 + }, + "decision": "REFUTED: C not constant across \u03b8" +} \ No newline at end of file diff --git a/data/e4_gamma/Qwen--Qwen2.5-7B_mongo.json b/data/e4_gamma/Qwen--Qwen2.5-7B_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..8ee26e391142a54450f810c4f06a13c6dbabb4ae --- /dev/null +++ b/data/e4_gamma/Qwen--Qwen2.5-7B_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "Qwen/Qwen2.5-7B", + "theta": 1000000, + "gamma_pred": 0.6666637444455867, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.01053441942591841, + "std_across_seeds": 0.0004096349139475821, + "per_seed_means": [ + 0.010172064562793822, + 0.01110703696652005, + 0.010324156748441358 + ] + }, + "20": { + "mean_across_seeds": 0.005741046461561281, + "std_across_seeds": 0.000435607874390518, + "per_seed_means": [ + 0.005440830650040879, + 0.006357023641079043, + 0.005425285093563919 + ] + }, + "30": { + "mean_across_seeds": 0.003944915988637755, + "std_across_seeds": 0.0002638345043073428, + "per_seed_means": [ + 0.003917800245884185, + 0.004280749395644913, + 0.0036361983243841677 + ] + }, + "50": { + "mean_across_seeds": 0.002418298812634829, + "std_across_seeds": 5.2016056601622415e-05, + "per_seed_means": [ + 0.0023832766552610942, + 0.0024918330257060004, + 0.002379786756937392 + ] + }, + "100": { + "mean_across_seeds": 0.0012223406427882666, + "std_across_seeds": 0.0001330805153728109, + "per_seed_means": [ + 0.0010793442847110177, + 0.0013998089428059756, + 0.0011878687008478057 + ] + }, + "200": { + "mean_across_seeds": 0.0004989972161840544, + "std_across_seeds": 4.554042469761155e-05, + "per_seed_means": [ + 0.0005171559086011257, + 0.0005434304090643612, + 0.0004364053308866763 + ] + }, + "500": { + "mean_across_seeds": 0.0002234310967226823, + "std_across_seeds": 1.850256370960218e-05, + "per_seed_means": [ + 0.00024275045480559736, + 0.00019848779578751419, + 0.0002290550395749354 + ] + }, + "1000": { + "mean_across_seeds": 0.00014713535652036727, + "std_across_seeds": 2.0011462308929375e-05, + "per_seed_means": [ + 0.0001648538616791484, + 0.00015738706589521218, + 0.00011916514198674121 + ] + }, + "2000": { + "mean_across_seeds": 5.419184616888136e-05, + "std_across_seeds": 7.110548031407352e-06, + "per_seed_means": [ + 6.415639788883709e-05, + 5.0380305857894806e-05, + 4.8038834759912184e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.9966953735480816, + "log_A": -2.1584093095473813, + "R2": 0.993942, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0019021924595511957, + "log_A": -6.506136394202672, + "R2": 0.780205, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2137, + "gamma_CI_95": { + "lo": 0.915955763320153, + "hi": 1.0588818264570377 + }, + "decision": "REFUTED: C not constant across \u03b8" +} \ No newline at end of file diff --git a/data/e4_gamma/Qwen--Qwen2.5-7B_random.json b/data/e4_gamma/Qwen--Qwen2.5-7B_random.json new file mode 100644 index 0000000000000000000000000000000000000000..0b89642ca418bb6608fa7258a2fddd1afb3db034 --- /dev/null +++ b/data/e4_gamma/Qwen--Qwen2.5-7B_random.json @@ -0,0 +1,134 @@ +{ + "model": "Qwen/Qwen2.5-7B", + "theta": 1000000, + "gamma_pred": 0.6666637444455867, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.009949791732554635, + "std_across_seeds": 0.00017400484474847187, + "per_seed_means": [ + 0.00970459462919583, + 0.010054353810846805, + 0.010090426757621269 + ] + }, + "20": { + "mean_across_seeds": 0.00652512194097249, + "std_across_seeds": 0.00027764715284577684, + "per_seed_means": [ + 0.006915346873768916, + 0.006367762469065686, + 0.00629225648008287 + ] + }, + "30": { + "mean_across_seeds": 0.004573273731706043, + "std_across_seeds": 0.0002440010861132137, + "per_seed_means": [ + 0.004245316321806361, + 0.00483019192237407, + 0.004644312950937699 + ] + }, + "50": { + "mean_across_seeds": 0.0037503224599640815, + "std_across_seeds": 0.00014970965595137803, + "per_seed_means": [ + 0.0036074511027739693, + 0.003686442569984744, + 0.003957073707133532 + ] + }, + "100": { + "mean_across_seeds": 0.0017626451529536603, + "std_across_seeds": 6.538491081934697e-05, + "per_seed_means": [ + 0.0016974271783449998, + 0.0017384849436348304, + 0.0018520233368811507 + ] + }, + "200": { + "mean_across_seeds": 0.0009006613932110162, + "std_across_seeds": 1.4542795753788803e-05, + "per_seed_means": [ + 0.0008808788963748763, + 0.0009056814018792162, + 0.0009154238813789561 + ] + }, + "500": { + "mean_across_seeds": 0.0004466055328116959, + "std_across_seeds": 2.9333787485462512e-05, + "per_seed_means": [ + 0.0004095632961252704, + 0.00044895225291838867, + 0.0004813010493914286 + ] + }, + "1000": { + "mean_across_seeds": 0.00036055122376031553, + "std_across_seeds": 3.763717614004054e-05, + "per_seed_means": [ + 0.00032056129857664927, + 0.00035012514175226294, + 0.0004109672309520344 + ] + }, + "2000": { + "mean_across_seeds": 0.00013122257518666125, + "std_across_seeds": 2.1018759696526725e-06, + "per_seed_means": [ + 0.0001299849769566208, + 0.00013418190562030456, + 0.00012950084298305833 + ] + } + }, + "fit_power_law": { + "gamma": 0.8270155146179213, + "log_A": -2.5043057610030255, + "R2": 0.984663, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.00159574430485393, + "log_A": -6.102228480230107, + "R2": 0.790044, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.1946, + "gamma_CI_95": { + "lo": 0.718555951342085, + "hi": 0.9106818877708862 + }, + "decision": "UNCLEAR: \u03b3=0.827 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/allenai--OLMo-7B-hf_mongo.json b/data/e4_gamma/allenai--OLMo-7B-hf_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..5e9fff0b853f6f0e4405d8f0ca76dce00448ad80 --- /dev/null +++ b/data/e4_gamma/allenai--OLMo-7B-hf_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "allenai/OLMo-7B-hf", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 30, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.009969798744552666, + "std_across_seeds": 0.0008748989663154168, + "per_seed_means": [ + 0.0088797216458867, + 0.011021760602792105, + 0.010007913984979192 + ] + }, + "20": { + "mean_across_seeds": 0.007337944313055939, + "std_across_seeds": 0.0005311704919325939, + "per_seed_means": [ + 0.006588618985066811, + 0.007758397826304038, + 0.007666816127796968 + ] + }, + "30": { + "mean_across_seeds": 0.005870271401686801, + "std_across_seeds": 0.0004457334001566012, + "per_seed_means": [ + 0.005248625266055266, + 0.006271562461430828, + 0.006090626477574309 + ] + }, + "50": { + "mean_across_seeds": 0.004284175974316895, + "std_across_seeds": 0.00013979457191493625, + "per_seed_means": [ + 0.0040915545852233965, + 0.0044190422631800175, + 0.0043419310745472705 + ] + }, + "100": { + "mean_across_seeds": 0.0027131779248722727, + "std_across_seeds": 9.807202554892205e-05, + "per_seed_means": [ + 0.0025766180517772836, + 0.0027604641936098536, + 0.0028024515292296806 + ] + }, + "200": { + "mean_across_seeds": 0.001480693245927493, + "std_across_seeds": 8.252413292374016e-05, + "per_seed_means": [ + 0.0013760605710558594, + 0.0015777795653169355, + 0.0014882396014096837 + ] + }, + "500": { + "mean_across_seeds": 0.0006563582910328276, + "std_across_seeds": 1.4485757765891709e-05, + "per_seed_means": [ + 0.0006361316579083601, + 0.0006636576688227554, + 0.0006692855463673671 + ] + }, + "1000": { + "mean_across_seeds": 0.00032767843028220036, + "std_across_seeds": 6.507579470295881e-06, + "per_seed_means": [ + 0.0003185212437529117, + 0.0003330522197453926, + 0.0003314618273482968 + ] + }, + "2000": { + "mean_across_seeds": 0.00020018271364582083, + "std_across_seeds": 4.4346897957223284e-07, + "per_seed_means": [ + 0.00020006364405465623, + 0.00019970899059747657, + 0.00020077550628532965 + ] + } + }, + "fit_power_law": { + "gamma": 0.8291935914574181, + "log_A": -2.203505644417151, + "R2": 0.99587, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0016030435200460648, + "log_A": -5.8091876812482885, + "R2": 0.802132, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.1937, + "gamma_CI_95": { + "lo": 0.7599769664697463, + "hi": 0.9074527203730529 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/allenai--OLMo-7B-hf_random.json b/data/e4_gamma/allenai--OLMo-7B-hf_random.json new file mode 100644 index 0000000000000000000000000000000000000000..5fa0902d1b4577cba62edb7d1d47354269f388b7 --- /dev/null +++ b/data/e4_gamma/allenai--OLMo-7B-hf_random.json @@ -0,0 +1,134 @@ +{ + "model": "allenai/OLMo-7B-hf", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 30, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.011950965768968065, + "std_across_seeds": 0.0019094740885650225, + "per_seed_means": [ + 0.01029754610111316, + 0.0109286787143598, + 0.014626672491431236 + ] + }, + "20": { + "mean_across_seeds": 0.00918573875290652, + "std_across_seeds": 0.0017129117350548116, + "per_seed_means": [ + 0.008228305944552023, + 0.007737386987234155, + 0.011591523326933384 + ] + }, + "30": { + "mean_across_seeds": 0.007727537997480896, + "std_across_seeds": 0.0008311603361518234, + "per_seed_means": [ + 0.007860692869871854, + 0.006649553822353482, + 0.00867236730021735 + ] + }, + "50": { + "mean_across_seeds": 0.006277959416103031, + "std_across_seeds": 0.0010658756396334493, + "per_seed_means": [ + 0.0049349239251265924, + 0.006356747820973396, + 0.007542206502209107 + ] + }, + "100": { + "mean_across_seeds": 0.004609536032916771, + "std_across_seeds": 0.0011788744650315327, + "per_seed_means": [ + 0.005834931616360942, + 0.003017848070400457, + 0.004975828411988914 + ] + }, + "200": { + "mean_across_seeds": 0.0022158400103863742, + "std_across_seeds": 0.0006627513905740745, + "per_seed_means": [ + 0.0016097930609248578, + 0.0018996789973850052, + 0.0031380479728492596 + ] + }, + "500": { + "mean_across_seeds": 0.001623525357960413, + "std_across_seeds": 0.0007308751154157786, + "per_seed_means": [ + 0.000765473044399793, + 0.0015534750457542638, + 0.002551627983727182 + ] + }, + "1000": { + "mean_across_seeds": 0.000880023775001367, + "std_across_seeds": 0.0006987926051610056, + "per_seed_means": [ + 0.0003818664855013291, + 0.0003899501054547727, + 0.001868254734047999 + ] + }, + "2000": { + "mean_across_seeds": 0.0009936139660163058, + "std_across_seeds": 0.0005670452679431608, + "per_seed_means": [ + 0.0016314704378601163, + 0.00025377618536973995, + 0.0010955952748190612 + ] + } + }, + "fit_power_law": { + "gamma": 0.550891782676269, + "log_A": -2.9853551451655087, + "R2": 0.957841, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0009728174662317398, + "log_A": -5.431967719378075, + "R2": 0.643706, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.3141, + "gamma_CI_95": { + "lo": 0.4640345252970487, + "hi": 0.6819655922489186 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/allenai--OLMo-7B_mongo.json b/data/e4_gamma/allenai--OLMo-7B_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..c081fe87b23bdb9ea7a6d17fbffce8583c0da9a6 --- /dev/null +++ b/data/e4_gamma/allenai--OLMo-7B_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "allenai/OLMo-7B", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.01039986034648286, + "std_across_seeds": 0.0008646857912582376, + "per_seed_means": [ + 0.00994157621016105, + 0.009647165353720387, + 0.011610839475567142 + ] + }, + "20": { + "mean_across_seeds": 0.007648689849302173, + "std_across_seeds": 0.0006192345467580065, + "per_seed_means": [ + 0.007250080776090424, + 0.007172709082563718, + 0.008523279689252377 + ] + }, + "30": { + "mean_across_seeds": 0.006203431694044008, + "std_across_seeds": 0.0007375300473230609, + "per_seed_means": [ + 0.005719005291660626, + 0.005645692301914096, + 0.007245597488557299 + ] + }, + "50": { + "mean_across_seeds": 0.0047963064427798, + "std_across_seeds": 0.0007803886881250668, + "per_seed_means": [ + 0.004292445137786369, + 0.00419788270102193, + 0.0058985914895311 + ] + }, + "100": { + "mean_across_seeds": 0.00310110789258033, + "std_across_seeds": 0.0006727254324228533, + "per_seed_means": [ + 0.002647494715638459, + 0.0026036800056075055, + 0.004052148956495027 + ] + }, + "200": { + "mean_across_seeds": 0.0018887472636480301, + "std_across_seeds": 0.0006434028538092133, + "per_seed_means": [ + 0.0014282328449189664, + 0.0014393753822272023, + 0.002798633563797921 + ] + }, + "500": { + "mean_across_seeds": 0.0011960331072461688, + "std_across_seeds": 0.0007723658926822139, + "per_seed_means": [ + 0.0006483791035134345, + 0.0006513981812167912, + 0.0022883220370082807 + ] + }, + "1000": { + "mean_across_seeds": 0.0008328753268708371, + "std_across_seeds": 0.0007111226928837001, + "per_seed_means": [ + 0.00033069968689233067, + 0.0003293719022379567, + 0.001838554391482224 + ] + }, + "2000": { + "mean_across_seeds": 0.0007975042867474258, + "std_across_seeds": 0.0008420581838313258, + "per_seed_means": [ + 0.00020227690887016556, + 0.00020188158261589707, + 0.0019883543687562147 + ] + } + }, + "fit_power_law": { + "gamma": 0.5243966907611701, + "log_A": -3.361171045437037, + "R2": 0.976863, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0009215067015090179, + "log_A": -5.692621144600664, + "R2": 0.650092, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.3268, + "gamma_CI_95": { + "lo": 0.44705152948588966, + "hi": 0.6152026054827997 + }, + "decision": "UNCLEAR: \u03b3=0.524 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/allenai--OLMo-7B_random.json b/data/e4_gamma/allenai--OLMo-7B_random.json new file mode 100644 index 0000000000000000000000000000000000000000..10b92337f4cbaba2e500dfa84aa5bccae040863f --- /dev/null +++ b/data/e4_gamma/allenai--OLMo-7B_random.json @@ -0,0 +1,134 @@ +{ + "model": "allenai/OLMo-7B", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 1950 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 1950 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.011078201784855789, + "std_across_seeds": 0.0004568002389428613, + "per_seed_means": [ + 0.01044130727648735, + 0.011490317210555077, + 0.011302980867524941 + ] + }, + "20": { + "mean_across_seeds": 0.008723362791869376, + "std_across_seeds": 0.0009938605218391735, + "per_seed_means": [ + 0.00813545612928768, + 0.007911688508465886, + 0.010122943737854561 + ] + }, + "30": { + "mean_across_seeds": 0.007300337539571854, + "std_across_seeds": 0.00046900116814976243, + "per_seed_means": [ + 0.006724534671132763, + 0.007303139980261525, + 0.007873337967321276 + ] + }, + "50": { + "mean_across_seeds": 0.005650945112316144, + "std_across_seeds": 0.00027488861469437606, + "per_seed_means": [ + 0.006032803847144047, + 0.005396900329117974, + 0.0055231311606864135 + ] + }, + "100": { + "mean_across_seeds": 0.004058174822582967, + "std_across_seeds": 0.0005285617905847457, + "per_seed_means": [ + 0.0040144564071670176, + 0.004726279242895544, + 0.0034337888176863393 + ] + }, + "200": { + "mean_across_seeds": 0.001906132831548651, + "std_across_seeds": 0.000362056287267553, + "per_seed_means": [ + 0.0016092082873607676, + 0.0016933417382339637, + 0.002415848469051222 + ] + }, + "500": { + "mean_across_seeds": 0.0015021604990276196, + "std_across_seeds": 0.000235915454698382, + "per_seed_means": [ + 0.0014356055110692977, + 0.0012523090862669051, + 0.0018185668997466565 + ] + }, + "1000": { + "mean_across_seeds": 0.0013560072134714575, + "std_across_seeds": 0.0008188443016643402, + "per_seed_means": [ + 0.0003828349825926125, + 0.0012990302301477642, + 0.0023861564276739956 + ] + }, + "1950": { + "mean_across_seeds": 0.0006863480689935387, + "std_across_seeds": 0.00031650373924358707, + "per_seed_means": [ + 0.0009987773909233512, + 0.0002525494305882603, + 0.0008077173854690045 + ] + } + }, + "fit_power_law": { + "gamma": 0.5411598042384096, + "log_A": -3.1051096313732693, + "R2": 0.96368, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0010459025390052495, + "log_A": -5.463978356707257, + "R2": 0.746322, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2174, + "gamma_CI_95": { + "lo": 0.448109235206576, + "hi": 0.6342278689106579 + }, + "decision": "UNCLEAR: \u03b3=0.541 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/bigscience--bloom-7b1_mongo.json b/data/e4_gamma/bigscience--bloom-7b1_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..d4815c96a39a0c267026237c9e59e384daa00f3e --- /dev/null +++ b/data/e4_gamma/bigscience--bloom-7b1_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "bigscience/bloom-7b1", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.017026184912926207, + "std_across_seeds": 0.0007541454672392034, + "per_seed_means": [ + 0.01766309896406407, + 0.01596687807313477, + 0.01744857770157978 + ] + }, + "20": { + "mean_across_seeds": 0.008486570799205867, + "std_across_seeds": 0.00024768736323497326, + "per_seed_means": [ + 0.008657664684578776, + 0.008665728799145048, + 0.008136318913893775 + ] + }, + "30": { + "mean_across_seeds": 0.00567762653925456, + "std_across_seeds": 0.00024844833070221883, + "per_seed_means": [ + 0.006013596208843713, + 0.005598709910797576, + 0.005420573498122394 + ] + }, + "50": { + "mean_across_seeds": 0.00330956853466988, + "std_across_seeds": 0.0003124236971576066, + "per_seed_means": [ + 0.003732492331570635, + 0.0029873616754775866, + 0.0032088515969614186 + ] + }, + "100": { + "mean_across_seeds": 0.0013943244779107368, + "std_across_seeds": 7.40627584011253e-05, + "per_seed_means": [ + 0.001463271676717947, + 0.0014281344863896568, + 0.0012915672706246065 + ] + }, + "200": { + "mean_across_seeds": 0.0006065161687082663, + "std_across_seeds": 3.2625580332948275e-05, + "per_seed_means": [ + 0.0006210775734022415, + 0.0005613195480934034, + 0.0006371513846291539 + ] + }, + "500": { + "mean_across_seeds": 0.0001585892659477395, + "std_across_seeds": 1.4564797014947626e-06, + "per_seed_means": [ + 0.00015803095979814922, + 0.00016058545727598054, + 0.00015715138076908866 + ] + }, + "1000": { + "mean_across_seeds": 7.755193713743817e-05, + "std_across_seeds": 6.7177280959339004e-06, + "per_seed_means": [ + 8.093510436386471e-05, + 8.354848989104843e-05, + 6.817221715740137e-05 + ] + }, + "2000": { + "mean_across_seeds": 3.9846021137843006e-05, + "std_across_seeds": 2.6795868885364806e-06, + "per_seed_means": [ + 4.158550534157257e-05, + 3.606064447922108e-05, + 4.189191359273536e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.2178239236386044, + "log_A": -1.0007599629339874, + "R2": 0.99733, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.002279463727406832, + "log_A": -6.3378868865044415, + "R2": 0.753007, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2443, + "gamma_CI_95": { + "lo": 1.18199000827358, + "hi": 1.295463995171343 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/bigscience--bloom-7b1_random.json b/data/e4_gamma/bigscience--bloom-7b1_random.json new file mode 100644 index 0000000000000000000000000000000000000000..f7c3a814208a579d9fe809efccfaf444851465cb --- /dev/null +++ b/data/e4_gamma/bigscience--bloom-7b1_random.json @@ -0,0 +1,134 @@ +{ + "model": "bigscience/bloom-7b1", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.01273650299343798, + "std_across_seeds": 0.0001509585485643521, + "per_seed_means": [ + 0.012947040585180123, + 0.012600604686886072, + 0.01266186370824774 + ] + }, + "20": { + "mean_across_seeds": 0.007338399014228748, + "std_across_seeds": 0.00021254883175654316, + "per_seed_means": [ + 0.007350003939742843, + 0.007072472536625961, + 0.007592720566317439 + ] + }, + "30": { + "mean_across_seeds": 0.005911933845685175, + "std_across_seeds": 0.0004378509688092584, + "per_seed_means": [ + 0.005785020792391151, + 0.0054505189461633565, + 0.006500261798501015 + ] + }, + "50": { + "mean_across_seeds": 0.0031548952059044194, + "std_across_seeds": 0.0001435271512111273, + "per_seed_means": [ + 0.002965987913388138, + 0.0033136574116845925, + 0.003185040292640527 + ] + }, + "100": { + "mean_across_seeds": 0.0016561893898890251, + "std_across_seeds": 1.9276906361242315e-05, + "per_seed_means": [ + 0.0016569003967257837, + 0.0016322326252702624, + 0.001679435147671029 + ] + }, + "200": { + "mean_across_seeds": 0.0008414794023459156, + "std_across_seeds": 4.253952973019571e-05, + "per_seed_means": [ + 0.0008747415770388519, + 0.0008682607347145677, + 0.0007814358952843274 + ] + }, + "500": { + "mean_across_seeds": 0.00040051930986616447, + "std_across_seeds": 3.5547961628649225e-05, + "per_seed_means": [ + 0.0003784106351182951, + 0.00037247260838436583, + 0.0004506746860958325 + ] + }, + "1000": { + "mean_across_seeds": 0.00022698354934820803, + "std_across_seeds": 1.549507336493239e-05, + "per_seed_means": [ + 0.00021698228888756906, + 0.00021509847438816603, + 0.000248869884768889 + ] + }, + "2000": { + "mean_across_seeds": 0.0001414372526051011, + "std_across_seeds": 2.055481036509795e-05, + "per_seed_means": [ + 0.0001610486341814976, + 0.0001130494298558915, + 0.00015021369377791417 + ] + } + }, + "fit_power_law": { + "gamma": 0.882390442263598, + "log_A": -2.2817618434777787, + "R2": 0.994632, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0016233437384084728, + "log_A": -6.164518491356999, + "R2": 0.725481, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2692, + "gamma_CI_95": { + "lo": 0.8122211931161655, + "hi": 0.9731666058285073 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/cdr_analysis_2026-04-18.json b/data/e4_gamma/cdr_analysis_2026-04-18.json new file mode 100644 index 0000000000000000000000000000000000000000..308f21d976872d275304e85cc23e071f60a41c7a --- /dev/null +++ b/data/e4_gamma/cdr_analysis_2026-04-18.json @@ -0,0 +1,47 @@ +{ + "experiment": "CDR_theta_analysis_2026-04-18", + "conclusion": "NEW FORMULA VALIDATED: gamma = 1 - T_eval*sqrt(2)/theta", + "old_formula": { + "formula": "gamma = C/ln(theta)", + "C": 9.2103, + "mean_error_pct": 33.3, + "status": "REFUTED" + }, + "new_formula": { + "formula": "gamma = 1 - T_eval*sqrt(2)/theta", + "T_eval_e4": 2000, + "mean_error_pct": 3.0, + "status": "CONFIRMED" + }, + "data": [ + { + "model": "pythia-70m", + "theta": 10000, + "gamma_obs": 0.7476, + "gamma_old": 1.0, + "gamma_new": 0.7172, + "err_old_pct": 33.8, + "err_new_pct": 4.1 + }, + { + "model": "Llama-3-8B", + "theta": 500000, + "gamma_obs": 1.0455, + "gamma_old": 0.7019, + "gamma_new": 0.9943, + "err_old_pct": 32.9, + "err_new_pct": 4.9 + }, + { + "model": "Qwen2.5-7B", + "theta": 1000000, + "gamma_obs": 0.9967, + "gamma_old": 0.6667, + "gamma_new": 0.9972, + "err_old_pct": 33.1, + "err_new_pct": 0.0 + } + ], + "note": "gamma>1 for Llama-3 likely from long-context fine-tuning (extra positional sharpening)", + "new_physics": "gamma = 1 - T_eval*sqrt(2)/theta = fraction of RoPE phases NOT aliased at distance T_eval" +} \ No newline at end of file diff --git a/data/e4_gamma/cerebras--Cerebras-GPT-2.7B_mongo.json b/data/e4_gamma/cerebras--Cerebras-GPT-2.7B_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..6ca5ef47403dd9e97f56b33ef503d0ea3958c3c0 --- /dev/null +++ b/data/e4_gamma/cerebras--Cerebras-GPT-2.7B_mongo.json @@ -0,0 +1,90 @@ +{ + "model": "cerebras/Cerebras-GPT-2.7B", + "theta": null, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.011066294480632578, + "std_across_seeds": 0.0003957905898929582, + "per_seed_means": [ + 0.010601912275888026, + 0.01102786192825685, + 0.011569109237752855 + ] + }, + "20": { + "mean_across_seeds": 0.007225409053110828, + "std_across_seeds": 0.0005509117999463269, + "per_seed_means": [ + 0.006504136952571571, + 0.007330936039021859, + 0.007841154167739053 + ] + }, + "30": { + "mean_across_seeds": 0.005252867244934249, + "std_across_seeds": 0.0002543120148813196, + "per_seed_means": [ + 0.005113226118652771, + 0.005035656157803411, + 0.0056097194583465656 + ] + }, + "50": { + "mean_across_seeds": 0.003439449271989159, + "std_across_seeds": 8.001153093620939e-05, + "per_seed_means": [ + 0.0033557674067560585, + 0.0034153297869488597, + 0.0035472506222625575 + ] + }, + "100": { + "mean_across_seeds": 0.0020080710809108697, + "std_across_seeds": 0.0002751025739235102, + "per_seed_means": [ + 0.0019321673724334687, + 0.0017155670234933495, + 0.002376478846805791 + ] + }, + "200": { + "mean_across_seeds": 0.0008928673497090736, + "std_across_seeds": 9.575487302045444e-05, + "per_seed_means": [ + 0.0008424188293671857, + 0.0008092582342214882, + 0.0010269249855385473 + ] + }, + "500": { + "mean_across_seeds": 0.00032154169373421206, + "std_across_seeds": 3.624580892715188e-05, + "per_seed_means": [ + 0.0002953432806922744, + 0.00029648503094601136, + 0.00037279676956435045 + ] + }, + "1000": { + "mean_across_seeds": 0.00014317721089658638, + "std_across_seeds": 1.7442142724204942e-05, + "per_seed_means": [ + 0.00013669004430994392, + 0.00012581060213657716, + 0.00016703098624323804 + ] + } + }, + "runtime_so_far": 433.1 +} \ No newline at end of file diff --git a/data/e4_gamma/cerebras--Cerebras-GPT-2.7B_mongo.partial.json b/data/e4_gamma/cerebras--Cerebras-GPT-2.7B_mongo.partial.json new file mode 100644 index 0000000000000000000000000000000000000000..6ca5ef47403dd9e97f56b33ef503d0ea3958c3c0 --- /dev/null +++ b/data/e4_gamma/cerebras--Cerebras-GPT-2.7B_mongo.partial.json @@ -0,0 +1,90 @@ +{ + "model": "cerebras/Cerebras-GPT-2.7B", + "theta": null, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.011066294480632578, + "std_across_seeds": 0.0003957905898929582, + "per_seed_means": [ + 0.010601912275888026, + 0.01102786192825685, + 0.011569109237752855 + ] + }, + "20": { + "mean_across_seeds": 0.007225409053110828, + "std_across_seeds": 0.0005509117999463269, + "per_seed_means": [ + 0.006504136952571571, + 0.007330936039021859, + 0.007841154167739053 + ] + }, + "30": { + "mean_across_seeds": 0.005252867244934249, + "std_across_seeds": 0.0002543120148813196, + "per_seed_means": [ + 0.005113226118652771, + 0.005035656157803411, + 0.0056097194583465656 + ] + }, + "50": { + "mean_across_seeds": 0.003439449271989159, + "std_across_seeds": 8.001153093620939e-05, + "per_seed_means": [ + 0.0033557674067560585, + 0.0034153297869488597, + 0.0035472506222625575 + ] + }, + "100": { + "mean_across_seeds": 0.0020080710809108697, + "std_across_seeds": 0.0002751025739235102, + "per_seed_means": [ + 0.0019321673724334687, + 0.0017155670234933495, + 0.002376478846805791 + ] + }, + "200": { + "mean_across_seeds": 0.0008928673497090736, + "std_across_seeds": 9.575487302045444e-05, + "per_seed_means": [ + 0.0008424188293671857, + 0.0008092582342214882, + 0.0010269249855385473 + ] + }, + "500": { + "mean_across_seeds": 0.00032154169373421206, + "std_across_seeds": 3.624580892715188e-05, + "per_seed_means": [ + 0.0002953432806922744, + 0.00029648503094601136, + 0.00037279676956435045 + ] + }, + "1000": { + "mean_across_seeds": 0.00014317721089658638, + "std_across_seeds": 1.7442142724204942e-05, + "per_seed_means": [ + 0.00013669004430994392, + 0.00012581060213657716, + 0.00016703098624323804 + ] + } + }, + "runtime_so_far": 433.1 +} \ No newline at end of file diff --git a/data/e4_gamma/codellama--CodeLlama-13b-Instruct-hf_mongo.partial.json b/data/e4_gamma/codellama--CodeLlama-13b-Instruct-hf_mongo.partial.json new file mode 100644 index 0000000000000000000000000000000000000000..89cf56b530d513441eeb6d9dcd6d83177d28869e --- /dev/null +++ b/data/e4_gamma/codellama--CodeLlama-13b-Instruct-hf_mongo.partial.json @@ -0,0 +1,80 @@ +{ + "model": "codellama/CodeLlama-13b-Instruct-hf", + "theta": 1000000, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.0048818717902112336, + "std_across_seeds": 0.000618221857803684, + "per_seed_means": [ + 0.004535025885949532, + 0.004360261766705662, + 0.005750327717978507 + ] + }, + "20": { + "mean_across_seeds": 0.0030417641121635425, + "std_across_seeds": 0.0005179837604637786, + "per_seed_means": [ + 0.0026620033088450633, + 0.00268915298084418, + 0.0037741360468013835 + ] + }, + "30": { + "mean_across_seeds": 0.002327263202039628, + "std_across_seeds": 0.0004959427119133391, + "per_seed_means": [ + 0.0017478886434885983, + 0.002274631038502169, + 0.002959269924128118 + ] + }, + "50": { + "mean_across_seeds": 0.0017201522036339155, + "std_across_seeds": 0.000589432136747203, + "per_seed_means": [ + 0.001112376618257258, + 0.001529973205178976, + 0.002518106787465513 + ] + }, + "100": { + "mean_across_seeds": 0.0011313119502544093, + "std_across_seeds": 0.0005389199726514419, + "per_seed_means": [ + 0.0006684767865226604, + 0.0008383348202914931, + 0.0018871242439490743 + ] + }, + "200": { + "mean_across_seeds": 0.0007407459302178015, + "std_across_seeds": 0.00044933672113790606, + "per_seed_means": [ + 0.00030548142841629064, + 0.0005574245135358069, + 0.001359331848701307 + ] + }, + "500": { + "mean_across_seeds": 0.0005816437313074453, + "std_across_seeds": 0.00045393043226084014, + "per_seed_means": [ + 0.00014946052552356075, + 0.0003866483440409259, + 0.0012088223243578492 + ] + } + }, + "runtime_so_far": 16.2 +} \ No newline at end of file diff --git a/data/e4_gamma/content_vs_pos_decay.json b/data/e4_gamma/content_vs_pos_decay.json new file mode 100644 index 0000000000000000000000000000000000000000..c421bb616d9ed2b30809dca372125a7395833bfe --- /dev/null +++ b/data/e4_gamma/content_vs_pos_decay.json @@ -0,0 +1,67 @@ +{ + "analysis": "T-ContentDecay: short vs long range gamma split", + "d_short_max": 100, + "d_long_min": 200, + "models": [ + { + "model": "EleutherAI/pythia-70m", + "theta": 10000, + "gamma_full": 0.6811, + "r2_full": 0.9825, + "gamma_short": 0.6611, + "r2_short": 0.9632, + "gamma_long": 0.8564, + "r2_long": 0.9815, + "delta_long_minus_short": 0.1953, + "n_short": 5, + "n_long": 4, + "short_range": "d<=100", + "long_range": "d>=200" + }, + { + "model": "meta-llama/Meta-Llama-3-8B", + "theta": 500000, + "gamma_full": 0.8313, + "r2_full": 0.9936, + "gamma_short": 0.8125, + "r2_short": 0.9899, + "gamma_long": 1.0413, + "r2_long": 0.988, + "delta_long_minus_short": 0.2289, + "n_short": 5, + "n_long": 4, + "short_range": "d<=100", + "long_range": "d>=200" + }, + { + "model": "mistralai/Mistral-7B-v0.1", + "theta": 10000, + "gamma_full": null, + "r2_full": null, + "gamma_short": null, + "r2_short": null, + "gamma_long": null, + "r2_long": null, + "delta_long_minus_short": null, + "n_short": 2, + "n_long": 0, + "short_range": "d<=100", + "long_range": "d>=200" + }, + { + "model": "Qwen/Qwen2.5-7B", + "theta": 1000000, + "gamma_full": 0.9145, + "r2_full": 0.9993, + "gamma_short": 0.9041, + "r2_short": 0.9996, + "gamma_long": 0.8542, + "r2_long": 0.9936, + "delta_long_minus_short": -0.0499, + "n_short": 5, + "n_long": 4, + "short_range": "d<=100", + "long_range": "d>=200" + } + ] +} \ No newline at end of file diff --git a/data/e4_gamma/deepseek-ai--deepseek-llm-7b-base_mongo.json b/data/e4_gamma/deepseek-ai--deepseek-llm-7b-base_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..93fb4efad6ece539fb0ae80e66c0fea989585f48 --- /dev/null +++ b/data/e4_gamma/deepseek-ai--deepseek-llm-7b-base_mongo.json @@ -0,0 +1,123 @@ +{ + "model": "deepseek-ai/deepseek-llm-7b-base", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 30, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.005312137135640822, + "std_across_seeds": 0.0012807888523913684, + "per_seed_means": [ + 0.0037275183286207417, + 0.00686429359872515, + 0.005344599479576573 + ] + }, + "20": { + "mean_across_seeds": 0.003641892545339134, + "std_across_seeds": 0.000819973338952788, + "per_seed_means": [ + 0.002705339123106872, + 0.004702356034734597, + 0.003517982478175933 + ] + }, + "30": { + "mean_across_seeds": 0.002515320484073729, + "std_across_seeds": 0.000691025730828852, + "per_seed_means": [ + 0.001872401766013354, + 0.0034741702120906362, + 0.0021993894741171973 + ] + }, + "50": { + "mean_across_seeds": 0.0015479444335344142, + "std_across_seeds": 0.0006471615268145907, + "per_seed_means": [ + 0.0009777121934651707, + 0.0024530230177333577, + 0.0012130980894047147 + ] + }, + "100": { + "mean_across_seeds": 0.0010048023609366887, + "std_across_seeds": 0.0003930820678956331, + "per_seed_means": [ + 0.0007488263378036209, + 0.0015601397317368537, + 0.000705441013269592 + ] + }, + "200": { + "mean_across_seeds": 0.000413751030484693, + "std_across_seeds": 0.00015512411350614638, + "per_seed_means": [ + 0.0002695003125457636, + 0.0006290162988686158, + 0.0003427364800396996 + ] + }, + "500": { + "mean_across_seeds": 0.00018154666375064861, + "std_across_seeds": 8.394357341258587e-05, + "per_seed_means": [ + 0.00012939172083861194, + 0.0002999804112429653, + 0.00011526785917036856 + ] + }, + "1000": { + "mean_across_seeds": 9.38364265190709e-05, + "std_across_seeds": 6.38745866612118e-05, + "per_seed_means": [ + 4.4527273591180956e-05, + 0.00018403793128527469, + 5.294407468075709e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.9469730981988765, + "log_A": -2.7178439863294157, + "R2": 0.995256, + "n_points": 6 + }, + "fit_exponential": { + "lambda": 0.0031087749651009904, + "log_A": -6.532141890858652, + "R2": 0.843791, + "n_points": 6 + }, + "delta_R2_power_minus_exp": 0.1515, + "gamma_CI_95": { + "lo": 0.8882660457746588, + "hi": 1.0302327538542175 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/deepseek-ai--deepseek-llm-7b-base_random.json b/data/e4_gamma/deepseek-ai--deepseek-llm-7b-base_random.json new file mode 100644 index 0000000000000000000000000000000000000000..3d46f6cc0da864a381031c20b01127e6dbadb777 --- /dev/null +++ b/data/e4_gamma/deepseek-ai--deepseek-llm-7b-base_random.json @@ -0,0 +1,123 @@ +{ + "model": "deepseek-ai/deepseek-llm-7b-base", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 30, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.005657422418395679, + "std_across_seeds": 0.0005846860773104111, + "per_seed_means": [ + 0.005110410096434256, + 0.005393927784947058, + 0.0064679293738057215 + ] + }, + "20": { + "mean_across_seeds": 0.004647593477016522, + "std_across_seeds": 0.00048082214110109684, + "per_seed_means": [ + 0.005324713497733076, + 0.004255031103578707, + 0.004363035829737782 + ] + }, + "30": { + "mean_across_seeds": 0.0034734817635681896, + "std_across_seeds": 0.00019801741058974787, + "per_seed_means": [ + 0.003752451972104609, + 0.003355164073097209, + 0.0033128292455027503 + ] + }, + "50": { + "mean_across_seeds": 0.0024960884869667807, + "std_across_seeds": 7.20555658221851e-05, + "per_seed_means": [ + 0.002579750999575481, + 0.002403874249042322, + 0.0025046402122825386 + ] + }, + "100": { + "mean_across_seeds": 0.0014252143646849111, + "std_across_seeds": 4.671877863194069e-05, + "per_seed_means": [ + 0.0014912832314924647, + 0.0013925607781857252, + 0.0013917990843765437 + ] + }, + "200": { + "mean_across_seeds": 0.0005560471006901935, + "std_across_seeds": 2.49004406196586e-05, + "per_seed_means": [ + 0.0005254053762958695, + 0.0005563392750142763, + 0.0005863966507604346 + ] + }, + "500": { + "mean_across_seeds": 0.00028741381490059815, + "std_across_seeds": 9.703038942466332e-06, + "per_seed_means": [ + 0.0002944421474239789, + 0.0002941062814594867, + 0.0002736930158183289 + ] + }, + "1000": { + "mean_across_seeds": 0.0001582873018277395, + "std_across_seeds": 1.3358025817751188e-05, + "per_seed_means": [ + 0.0001766652528507014, + 0.00014531104922449838, + 0.00015288560340801874 + ] + } + }, + "fit_power_law": { + "gamma": 0.9103448618583042, + "log_A": -2.498397902454579, + "R2": 0.992208, + "n_points": 6 + }, + "fit_exponential": { + "lambda": 0.002941553763216821, + "log_A": -6.179880671148576, + "R2": 0.814969, + "n_points": 6 + }, + "delta_R2_power_minus_exp": 0.1772, + "gamma_CI_95": { + "lo": 0.788328871639226, + "hi": 0.9995347983804295 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/dict6_entropy_gamma.json b/data/e4_gamma/dict6_entropy_gamma.json new file mode 100644 index 0000000000000000000000000000000000000000..f0b8d291e42a3f781471945d1734fb4278d04f4c --- /dev/null +++ b/data/e4_gamma/dict6_entropy_gamma.json @@ -0,0 +1,49 @@ +{ + "gpt2": { + "model": "gpt2", + "gamma_entropy": 0.5991175079638448, + "delta_H_mean": -0.580409634678587, + "delta_H_std": 0.6068057513831355, + "slope_H_logT": 0.4947840387896378, + "class": "B", + "H_by_layer": { + "1": 0.01259014970928314, + "3": -0.31659999551654955, + "5": -0.7319513880775332, + "7": -0.8846455254689319, + "10": -0.9814414140392033 + }, + "n_observations": 655 + }, + "gpt2-medium": { + "model": "gpt2-medium", + "gamma_entropy": 0.6240946879029537, + "delta_H_mean": -0.6818263993430178, + "delta_H_std": 0.6214868683292253, + "slope_H_logT": 0.4582884647747496, + "class": "B", + "H_by_layer": { + "1": 0.007492876543584501, + "6": -0.8091424135624263, + "11": -0.717433163548912, + "16": -0.9225663263289272, + "22": -0.9674829698184081 + }, + "n_observations": 655 + }, + "EleutherAI/pythia-70m": { + "model": "EleutherAI/pythia-70m", + "gamma_entropy": 0.5419393638930605, + "delta_H_mean": -0.4023633731941078, + "delta_H_std": 0.6330986192967964, + "slope_H_logT": 0.5024395148503386, + "class": "A", + "H_by_layer": { + "1": -0.15332401347135424, + "2": -0.08638924450027424, + "3": -0.6417974812677865, + "4": -0.9769821132597699 + }, + "n_observations": 660 + } +} \ No newline at end of file diff --git a/data/e4_gamma/expg4_gamma_validation.json b/data/e4_gamma/expg4_gamma_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..6f695a6a2bc8071f83ebcf02dc5e587f05aa5d14 --- /dev/null +++ b/data/e4_gamma/expg4_gamma_validation.json @@ -0,0 +1,143 @@ +{ + "experiment": "EXP-G4", + "formula": "gamma = 1 - T_eval*sqrt(2)/theta", + "date": "2026-04-18", + "results": [ + { + "prefix": "EleutherAI--pythia-70m", + "name": "pythia-70m", + "pe_type": "RoPE", + "notes": "", + "theta": 10000, + "d_head": 64, + "T_train": 2048, + "T_eval_max": 2000.0, + "n_points": 9, + "gamma_obs": 0.7476017873166874, + "gamma_new_pred": 0.717157287525381, + "gamma_old_pred": 0.99999561666838, + "err_new_pct": 4.245163553501351, + "err_old_pct": -25.239493568239496, + "R2": 0.9893049417040555 + }, + { + "prefix": "meta-llama--Meta-Llama-3-8B", + "name": "Meta-Llama-3-8B", + "pe_type": "RoPE", + "notes": "", + "theta": 500000, + "d_head": 128, + "T_train": 8192, + "T_eval_max": 2000.0, + "n_points": 9, + "gamma_obs": 1.0454762537473639, + "gamma_new_pred": 0.9943431457505076, + "gamma_old_pred": 0.7018781400200674, + "err_new_pct": 5.1424006104313325, + "err_old_pct": 48.95409817400391, + "R2": 0.996718622313285 + }, + { + "prefix": "Qwen--Qwen2.5-7B", + "name": "Qwen2.5-7B", + "pe_type": "RoPE", + "notes": "", + "theta": 1000000, + "d_head": 128, + "T_train": 8192, + "T_eval_max": 2000.0, + "n_points": 9, + "gamma_obs": 0.9966953735480816, + "gamma_new_pred": 0.9971715728752538, + "gamma_old_pred": 0.6666637444455867, + "err_new_pct": -0.047755004266634545, + "err_old_pct": 49.5049613620367, + "R2": 0.9963935840252253 + }, + { + "prefix": "meta-llama--Llama-2-7b-hf", + "name": "Llama-2-7b-hf", + "pe_type": "RoPE", + "notes": "artifact", + "theta": 10000, + "d_head": 128, + "T_train": 4096, + "T_eval_max": 2000.0, + "n_points": 9, + "gamma_obs": 0.2870574377368437, + "gamma_new_pred": 0.717157287525381, + "gamma_old_pred": 0.99999561666838, + "err_new_pct": -59.97287586278841, + "err_old_pct": -71.29413039896974, + "R2": 0.881768027724978 + }, + { + "prefix": "mistralai--Mistral-7B-v0.1", + "name": "Mistral-7B-v0.1", + "pe_type": "RoPE", + "notes": "only_2pts", + "theta": 10000, + "d_head": 128, + "T_train": 8192, + "T_eval_max": 50.0, + "n_points": 2, + "gamma_obs": 1.213076772373504, + "gamma_new_pred": 0.9929289321881345, + "gamma_old_pred": 0.99999561666838, + "err_new_pct": 22.17156062722697, + "err_old_pct": 21.30820897145856, + "R2": 1.0 + }, + { + "prefix": "mistralai--Mistral-Nemo-Instruct-2407", + "name": "Mistral-Nemo-Instruct-2407", + "pe_type": "RoPE", + "notes": "partial_7pts", + "theta": 1000000, + "d_head": 128, + "T_train": 131072, + "T_eval_max": 500.0, + "n_points": 7, + "gamma_obs": 0.5407084190220748, + "gamma_new_pred": 0.9992928932188134, + "gamma_old_pred": 0.6666637444455867, + "err_new_pct": -45.8908971842676, + "err_old_pct": -18.893381629483898, + "R2": 0.9671649502959694 + }, + { + "prefix": "google--gemma-2-9b-it", + "name": "gemma-2-9b-it", + "pe_type": "RoPE", + "notes": "partial", + "theta": 10000, + "d_head": 256, + "T_train": 8192, + "T_eval_max": 1000.0, + "n_points": 8, + "gamma_obs": 0.6586407289285032, + "gamma_new_pred": 0.8585786437626906, + "gamma_old_pred": 0.99999561666838, + "err_new_pct": -23.287082236050793, + "err_old_pct": -34.13563840181085, + "R2": 0.97730793078849 + }, + { + "prefix": "tiiuae--falcon-7b", + "name": "falcon-7b", + "pe_type": "ALiBi", + "notes": "no_rope", + "theta": 10000, + "d_head": 64, + "T_train": 2048, + "T_eval_max": 1000.0, + "n_points": 8, + "gamma_obs": 0.8928207115404576, + "gamma_new_pred": 0.8585786437626906, + "gamma_old_pred": 0.99999561666838, + "err_new_pct": 3.9882272901294638, + "err_old_pct": -10.717537491313212, + "R2": 0.9927613215692025 + } + ] +} \ No newline at end of file diff --git a/data/e4_gamma/expg4_new_gamma_analysis.json b/data/e4_gamma/expg4_new_gamma_analysis.json new file mode 100644 index 0000000000000000000000000000000000000000..ac1040fb570d210893f77cacabf4442e9eee6ff8 --- /dev/null +++ b/data/e4_gamma/expg4_new_gamma_analysis.json @@ -0,0 +1,140 @@ +{ + "experiment": "EXP-G4_gamma_new_formula_validation", + "date": "2026-04-18", + "formula": "gamma = 1 - T_eval*sqrt(2)/theta", + "models": [ + { + "prefix": "EleutherAI--pythia-70m", + "pe_type": "RoPE", + "notes": "", + "theta": 10000, + "d_head": 64, + "T_train": 2048, + "T_eval_max": 2000.0, + "n_points": 9, + "gamma_obs": 0.7476017873166874, + "gamma_new_pred": 0.717157287525381, + "gamma_old_pred": 0.99999561666838, + "err_new_pct": 4.245163553501351, + "err_old_pct": -25.239493568239496, + "R2": 0.9824939571028048 + }, + { + "prefix": "meta-llama--Meta-Llama-3-8B", + "pe_type": "RoPE", + "notes": "", + "theta": 500000, + "d_head": 128, + "T_train": 8192, + "T_eval_max": 2000.0, + "n_points": 9, + "gamma_obs": 1.0454762537473639, + "gamma_new_pred": 0.9943431457505076, + "gamma_old_pred": 0.7018781400200674, + "err_new_pct": 5.1424006104313325, + "err_old_pct": 48.95409817400391, + "R2": 0.9936044966891108 + }, + { + "prefix": "Qwen--Qwen2.5-7B", + "pe_type": "RoPE", + "notes": "", + "theta": 1000000, + "d_head": 128, + "T_train": 8192, + "T_eval_max": 2000.0, + "n_points": 9, + "gamma_obs": 0.9966953735480816, + "gamma_new_pred": 0.9971715728752538, + "gamma_old_pred": 0.6666637444455867, + "err_new_pct": -0.047755004266634545, + "err_old_pct": 49.5049613620367, + "R2": 0.9993151965766267 + }, + { + "prefix": "meta-llama--Llama-2-7b-hf", + "pe_type": "RoPE", + "notes": "known_artifact", + "theta": 10000, + "d_head": 128, + "T_train": 4096, + "T_eval_max": 2000.0, + "n_points": 9, + "gamma_obs": 0.2870574377368437, + "gamma_new_pred": 0.717157287525381, + "gamma_old_pred": 0.99999561666838, + "err_new_pct": -59.97287586278841, + "err_old_pct": -71.29413039896974, + "R2": 0.9665564682676285 + }, + { + "prefix": "mistralai--Mistral-7B-v0.1", + "pe_type": "RoPE", + "notes": "only_2pts", + "theta": 10000, + "d_head": 128, + "T_train": 8192, + "T_eval_max": 50.0, + "n_points": 2, + "gamma_obs": 1.213076772373502, + "gamma_new_pred": 0.9929289321881345, + "gamma_old_pred": 0.99999561666838, + "err_new_pct": 22.171560627226768, + "err_old_pct": 21.30820897145836, + "R2": 1.0 + }, + { + "prefix": "mistralai--Mistral-Nemo-Instruct-2407", + "pe_type": "RoPE", + "notes": "partial_7pts", + "theta": 1000000, + "d_head": 128, + "T_train": 131072, + "T_eval_max": 500.0, + "n_points": 7, + "gamma_obs": 0.6382710387667986, + "gamma_new_pred": 0.9992928932188134, + "gamma_old_pred": 0.6666637444455867, + "err_new_pct": -36.127731609211246, + "err_old_pct": -4.2589245200967385, + "R2": 0.9904023363048612 + }, + { + "prefix": "google--gemma-2-9b-it", + "pe_type": "RoPE", + "notes": "partial", + "theta": 10000, + "d_head": 256, + "T_train": 8192, + "T_eval_max": 1000.0, + "n_points": 8, + "gamma_obs": 0.7791408207653451, + "gamma_new_pred": 0.8585786437626906, + "gamma_old_pred": 0.99999561666838, + "err_new_pct": -9.25224772062953, + "err_old_pct": -22.085576398708866, + "R2": 0.9981402455707831 + }, + { + "prefix": "tiiuae--falcon-7b", + "pe_type": "ALiBi", + "notes": "exclude_no_rope", + "theta": 10000, + "d_head": 64, + "T_train": 2048, + "T_eval_max": 1000.0, + "n_points": 8, + "gamma_obs": 0.7865425131096547, + "gamma_new_pred": 0.8585786437626906, + "gamma_old_pred": 0.99999561666838, + "err_new_pct": -8.390161015109813, + "err_old_pct": -21.34540391985647, + "R2": 0.998318162512278 + } + ], + "summary": { + "n_clean": 5, + "mean_err_new_pct": 10.963059699608019, + "mean_err_old_pct": 30.008610804617142 + } +} \ No newline at end of file diff --git a/data/e4_gamma/google--gemma-2-9b-it_mongo.json b/data/e4_gamma/google--gemma-2-9b-it_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..a5e868262cafa5ce67e6860671163d8c082e71ee --- /dev/null +++ b/data/e4_gamma/google--gemma-2-9b-it_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "google/gemma-2-9b-it", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.017953182898410077, + "std_across_seeds": 0.0007041805794645068, + "per_seed_means": [ + 0.018890493793102602, + 0.017775889163992056, + 0.017193165738135577 + ] + }, + "20": { + "mean_across_seeds": 0.010547844236328577, + "std_across_seeds": 0.0005853188284703318, + "per_seed_means": [ + 0.010494978645195564, + 0.011289679798452804, + 0.009858874265337363 + ] + }, + "30": { + "mean_across_seeds": 0.007733995182853605, + "std_across_seeds": 0.0003677665847093681, + "per_seed_means": [ + 0.007786908143510421, + 0.008155621903715654, + 0.007259455501334742 + ] + }, + "50": { + "mean_across_seeds": 0.004784861977936493, + "std_across_seeds": 0.0002626832721511183, + "per_seed_means": [ + 0.004976073055683324, + 0.004965086993955386, + 0.004413425884170768 + ] + }, + "100": { + "mean_across_seeds": 0.0028697990643559023, + "std_across_seeds": 0.0001871648542152544, + "per_seed_means": [ + 0.0029843557137064635, + 0.0030191692445077933, + 0.00260587223485345 + ] + }, + "200": { + "mean_across_seeds": 0.0017543701516721437, + "std_across_seeds": 0.00021095385277890302, + "per_seed_means": [ + 0.0018175555651153748, + 0.0019752808996781825, + 0.0014702739902228738 + ] + }, + "500": { + "mean_across_seeds": 0.0011422816235285103, + "std_across_seeds": 0.0003238164820881095, + "per_seed_means": [ + 0.00077773795768735, + 0.0015645880537037253, + 0.0010845188591944558 + ] + }, + "1000": { + "mean_across_seeds": 0.000969841086035659, + "std_across_seeds": 0.00035551078144477604, + "per_seed_means": [ + 0.0006559841446869541, + 0.0014669198747469636, + 0.0007866192386730593 + ] + }, + "2000": { + "mean_across_seeds": 0.0004331948929355652, + "std_across_seeds": 0.00012259108285473552, + "per_seed_means": [ + 0.0003057853409260739, + 0.0005987228732495472, + 0.0003950764646310745 + ] + } + }, + "fit_power_law": { + "gamma": 0.6276459084140061, + "log_A": -2.864020841938658, + "R2": 0.977314, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0011958725625334026, + "log_A": -5.603004236734519, + "R2": 0.764607, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2127, + "gamma_CI_95": { + "lo": 0.49543651805241545, + "hi": 0.7237620691005047 + }, + "decision": "UNCLEAR: \u03b3=0.628 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/google--gemma-2-9b-it_random.json b/data/e4_gamma/google--gemma-2-9b-it_random.json new file mode 100644 index 0000000000000000000000000000000000000000..88f52be45febaa79ab5359ef14d572eb516e2324 --- /dev/null +++ b/data/e4_gamma/google--gemma-2-9b-it_random.json @@ -0,0 +1,134 @@ +{ + "model": "google/gemma-2-9b-it", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.01131202671935575, + "std_across_seeds": 0.0004384004663510598, + "per_seed_means": [ + 0.01191513662226498, + 0.011134915679382781, + 0.010886027856419483 + ] + }, + "20": { + "mean_across_seeds": 0.007342287618666887, + "std_across_seeds": 0.000120221270640251, + "per_seed_means": [ + 0.0071971141314134, + 0.007338237394578755, + 0.007491511330008507 + ] + }, + "30": { + "mean_across_seeds": 0.005608218560616176, + "std_across_seeds": 0.00012586673790719274, + "per_seed_means": [ + 0.005452117463573813, + 0.00576035007291163, + 0.005612188145363082 + ] + }, + "50": { + "mean_across_seeds": 0.004600468463678327, + "std_across_seeds": 0.00029442297838144305, + "per_seed_means": [ + 0.004994370874483138, + 0.004286649797577411, + 0.004520384718974432 + ] + }, + "100": { + "mean_across_seeds": 0.0025041243884091576, + "std_across_seeds": 0.00013270791219153536, + "per_seed_means": [ + 0.0026802454908223202, + 0.0024722163340387244, + 0.0023599113403664283 + ] + }, + "200": { + "mean_across_seeds": 0.0011723479800275526, + "std_across_seeds": 0.00010549223246344652, + "per_seed_means": [ + 0.00103700214473065, + 0.0012943720275264544, + 0.0011856697678255539 + ] + }, + "500": { + "mean_across_seeds": 0.00037003465085743104, + "std_across_seeds": 6.662525937622196e-05, + "per_seed_means": [ + 0.00031459913613313497, + 0.0004637339629213481, + 0.00033177085351780987 + ] + }, + "1000": { + "mean_across_seeds": 0.0001858278461845152, + "std_across_seeds": 5.3211896888022095e-06, + "per_seed_means": [ + 0.00019292063060371827, + 0.00018445904356970762, + 0.00018010386438011968 + ] + }, + "2000": { + "mean_across_seeds": 4.510536613654242e-05, + "std_across_seeds": 1.884703048218177e-06, + "per_seed_means": [ + 4.538706549889563e-05, + 4.266916409202774e-05, + 4.725986881870389e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.1347958464287666, + "log_A": -0.9640958037685541, + "R2": 0.976472, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.002322719608941013, + "log_A": -5.827234742708562, + "R2": 0.881621, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.0949, + "gamma_CI_95": { + "lo": 0.9244744962120605, + "hi": 1.331560490527999 + }, + "decision": "ANOMALY: long-context training effect" +} \ No newline at end of file diff --git a/data/e4_gamma/gpt2-large_mongo.json b/data/e4_gamma/gpt2-large_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..368b89ec975a1b3ef10094fa13fb025308966653 --- /dev/null +++ b/data/e4_gamma/gpt2-large_mongo.json @@ -0,0 +1,112 @@ +{ + "model": "gpt2-large", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.011807018478090564, + "std_across_seeds": 9.176815550307716e-05, + "per_seed_means": [ + 0.011677321566579243, + 0.011867851351077359, + 0.011875882516615094 + ] + }, + "20": { + "mean_across_seeds": 0.007371175814833905, + "std_across_seeds": 0.0005148817498044519, + "per_seed_means": [ + 0.006685100831867506, + 0.007502953601845851, + 0.00792547301078836 + ] + }, + "30": { + "mean_across_seeds": 0.005391007330682543, + "std_across_seeds": 0.00013638022316011367, + "per_seed_means": [ + 0.0054177829972468314, + 0.005212205937908341, + 0.0055430330568924545 + ] + }, + "50": { + "mean_across_seeds": 0.0033552472887095064, + "std_across_seeds": 0.00021185305352822194, + "per_seed_means": [ + 0.0032321954891085625, + 0.0031802012852858753, + 0.0036533450917340814 + ] + }, + "100": { + "mean_across_seeds": 0.0019032734972683504, + "std_across_seeds": 0.00024632934414648474, + "per_seed_means": [ + 0.0018501364922849462, + 0.0016316817027594273, + 0.0022280022967606783 + ] + }, + "200": { + "mean_across_seeds": 0.0009134268449916919, + "std_across_seeds": 0.00011711414438641881, + "per_seed_means": [ + 0.0009828733045530195, + 0.0007484865803659583, + 0.0010089206500560977 + ] + }, + "500": { + "mean_across_seeds": 0.0007072974517682774, + "std_across_seeds": 3.225522112231979e-05, + "per_seed_means": [ + 0.0006921701038178677, + 0.000677592218562495, + 0.0007521300329244695 + ] + } + }, + "fit_power_law": { + "gamma": 0.7526597997629645, + "log_A": -2.7601979390469125, + "R2": 0.963281, + "n_points": 5 + }, + "fit_exponential": { + "lambda": 0.003782551699198506, + "log_A": -5.621630767464056, + "R2": 0.727345, + "n_points": 5 + }, + "delta_R2_power_minus_exp": 0.2359, + "gamma_CI_95": { + "lo": 0.5412077262995896, + "hi": 0.9412467006888188 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/gpt2-large_random.json b/data/e4_gamma/gpt2-large_random.json new file mode 100644 index 0000000000000000000000000000000000000000..195abf65de46d9c0fd2bbe2268df7d267799cdb5 --- /dev/null +++ b/data/e4_gamma/gpt2-large_random.json @@ -0,0 +1,112 @@ +{ + "model": "gpt2-large", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.011413864347462853, + "std_across_seeds": 0.0001579435820555264, + "per_seed_means": [ + 0.011211261538167795, + 0.011596611064548294, + 0.01143372043967247 + ] + }, + "20": { + "mean_across_seeds": 0.007244661914495131, + "std_across_seeds": 0.00012217024587907557, + "per_seed_means": [ + 0.0070898931395883364, + 0.007255538233245412, + 0.0073885543706516425 + ] + }, + "30": { + "mean_across_seeds": 0.005716287524232433, + "std_across_seeds": 0.0003130617286701043, + "per_seed_means": [ + 0.005533202925386528, + 0.0054587288480252025, + 0.006156930799285571 + ] + }, + "50": { + "mean_across_seeds": 0.0037910339831271106, + "std_across_seeds": 3.9934645483000486e-05, + "per_seed_means": [ + 0.0037612097694848974, + 0.0037644123557644587, + 0.0038474798241319758 + ] + }, + "100": { + "mean_across_seeds": 0.0021022605025468187, + "std_across_seeds": 5.717774132946197e-05, + "per_seed_means": [ + 0.002033388592147579, + 0.0021000026418672253, + 0.002173390273625652 + ] + }, + "200": { + "mean_across_seeds": 0.0011218934138176133, + "std_across_seeds": 4.079493326923544e-05, + "per_seed_means": [ + 0.0011778938424928734, + 0.0010818805452436208, + 0.0011059058537163462 + ] + }, + "500": { + "mean_across_seeds": 0.0007943298560955251, + "std_across_seeds": 3.2178971501619564e-05, + "per_seed_means": [ + 0.0008341191658594957, + 0.000793562078421625, + 0.0007553083240054547 + ] + } + }, + "fit_power_law": { + "gamma": 0.7274410953080044, + "log_A": -2.7580282594251577, + "R2": 0.977312, + "n_points": 5 + }, + "fit_exponential": { + "lambda": 0.0037082468078250166, + "log_A": -5.514357261893173, + "R2": 0.759259, + "n_points": 5 + }, + "delta_R2_power_minus_exp": 0.2181, + "gamma_CI_95": { + "lo": 0.5546310485484207, + "hi": 0.8783278853409829 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/gpt2-medium_mongo.json b/data/e4_gamma/gpt2-medium_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..fc09266cbd2ba5cb64f4e9e5f84a60685053ae6a --- /dev/null +++ b/data/e4_gamma/gpt2-medium_mongo.json @@ -0,0 +1,112 @@ +{ + "model": "gpt2-medium", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.01058365886203117, + "std_across_seeds": 0.00033498650194487505, + "per_seed_means": [ + 0.01017508412245661, + 0.010580283172118166, + 0.010995609291518727 + ] + }, + "20": { + "mean_across_seeds": 0.006108071415219456, + "std_across_seeds": 0.0005854065109699566, + "per_seed_means": [ + 0.00541647755773738, + 0.006059755889388422, + 0.006847980798532566 + ] + }, + "30": { + "mean_across_seeds": 0.004540787913396748, + "std_across_seeds": 0.0002736234541719406, + "per_seed_means": [ + 0.004534102104759465, + 0.004209061918857818, + 0.0048791997165729606 + ] + }, + "50": { + "mean_across_seeds": 0.002821947819449835, + "std_across_seeds": 0.0002769741040311557, + "per_seed_means": [ + 0.0026393758598715067, + 0.0026131128383955608, + 0.0032133547600824387 + ] + }, + "100": { + "mean_across_seeds": 0.0017550596823113866, + "std_across_seeds": 0.00031265903948833314, + "per_seed_means": [ + 0.0017083045073862499, + 0.0013976565222643936, + 0.0021592180172835167 + ] + }, + "200": { + "mean_across_seeds": 0.0009152149715939435, + "std_across_seeds": 0.00011811715122664298, + "per_seed_means": [ + 0.0009794839961493077, + 0.0007495528910658322, + 0.00101660802756669 + ] + }, + "500": { + "mean_across_seeds": 0.0005035989970201627, + "std_across_seeds": 3.594791302830609e-05, + "per_seed_means": [ + 0.0004702701503022884, + 0.0004870178242951321, + 0.0005535090164630673 + ] + } + }, + "fit_power_law": { + "gamma": 0.7842197184521328, + "log_A": -2.7650033649005716, + "R2": 0.996208, + "n_points": 5 + }, + "fit_exponential": { + "lambda": 0.004166179275631, + "log_A": -5.706815825653799, + "R2": 0.840554, + "n_points": 5 + }, + "delta_R2_power_minus_exp": 0.1557, + "gamma_CI_95": { + "lo": 0.7440311683558056, + "hi": 0.8462771225454215 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/gpt2-medium_random.json b/data/e4_gamma/gpt2-medium_random.json new file mode 100644 index 0000000000000000000000000000000000000000..cafabdf50097511e7885d62c89b8c45db1d9e03c --- /dev/null +++ b/data/e4_gamma/gpt2-medium_random.json @@ -0,0 +1,112 @@ +{ + "model": "gpt2-medium", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.010287301251147357, + "std_across_seeds": 7.413211463795221e-05, + "per_seed_means": [ + 0.01027268440462649, + 0.010384515828142564, + 0.010204703520673016 + ] + }, + "20": { + "mean_across_seeds": 0.006385365477245715, + "std_across_seeds": 5.3300248045824584e-05, + "per_seed_means": [ + 0.006314447893140216, + 0.006398701953391234, + 0.006442946585205694 + ] + }, + "30": { + "mean_across_seeds": 0.005017658354093631, + "std_across_seeds": 0.00013432244038947871, + "per_seed_means": [ + 0.004931068173609674, + 0.004914528096560389, + 0.00520737879211083 + ] + }, + "50": { + "mean_across_seeds": 0.003488969567956196, + "std_across_seeds": 9.77337552851428e-05, + "per_seed_means": [ + 0.0035319206025451423, + 0.0033537213085219264, + 0.0035812667928015194 + ] + }, + "100": { + "mean_across_seeds": 0.002033369381679222, + "std_across_seeds": 7.857528258527185e-05, + "per_seed_means": [ + 0.00196459553631333, + 0.001992167077648143, + 0.002143345531076193 + ] + }, + "200": { + "mean_across_seeds": 0.0011699919848211315, + "std_across_seeds": 3.769924237989468e-05, + "per_seed_means": [ + 0.0012024838421105717, + 0.0011171392436760167, + 0.0011903528686768065 + ] + }, + "500": { + "mean_across_seeds": 0.0006411757039475358, + "std_across_seeds": 1.092356652683099e-05, + "per_seed_means": [ + 0.0006547514341461162, + 0.0006407722824951634, + 0.0006280033952013279 + ] + } + }, + "fit_power_law": { + "gamma": 0.7409610966104535, + "log_A": -2.778454721629106, + "R2": 0.9987, + "n_points": 5 + }, + "fit_exponential": { + "lambda": 0.003953099002532501, + "log_A": -5.555048022499108, + "R2": 0.849835, + "n_points": 5 + }, + "delta_R2_power_minus_exp": 0.1489, + "gamma_CI_95": { + "lo": 0.7037655202734364, + "hi": 0.7881511837034664 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/gpt2-xl_mongo.json b/data/e4_gamma/gpt2-xl_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..247f834770bf0a6a1e0f88300f82dd28302cc6d7 --- /dev/null +++ b/data/e4_gamma/gpt2-xl_mongo.json @@ -0,0 +1,112 @@ +{ + "model": "gpt2-xl", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.01143325393916004, + "std_across_seeds": 0.00028220017499024677, + "per_seed_means": [ + 0.011112850472951928, + 0.011387393491653105, + 0.011799517852875094 + ] + }, + "20": { + "mean_across_seeds": 0.0074049248212638, + "std_across_seeds": 0.0005527441844026533, + "per_seed_means": [ + 0.006750149144791067, + 0.007362527843409529, + 0.008102097475590805 + ] + }, + "30": { + "mean_across_seeds": 0.005560923445348938, + "std_across_seeds": 0.00017701828678329116, + "per_seed_means": [ + 0.005659976066090167, + 0.005312287505560865, + 0.005710506764395783 + ] + }, + "50": { + "mean_across_seeds": 0.0036900683108251537, + "std_across_seeds": 0.00025448060516157386, + "per_seed_means": [ + 0.0035762571298982946, + 0.0034512953107090046, + 0.004042652491868163 + ] + }, + "100": { + "mean_across_seeds": 0.0021142097505637346, + "std_across_seeds": 0.0002971316336979487, + "per_seed_means": [ + 0.002120053345958392, + 0.001747412698265786, + 0.0024751632074670247 + ] + }, + "200": { + "mean_across_seeds": 0.0011156960537644207, + "std_across_seeds": 0.00013747995992569522, + "per_seed_means": [ + 0.0011808323829124371, + 0.0009244803142307016, + 0.0012417754641501234 + ] + }, + "500": { + "mean_across_seeds": 0.0003066435451263614, + "std_across_seeds": 4.906065991413612e-05, + "per_seed_means": [ + 0.00027769860421055154, + 0.00026650772570671204, + 0.0003757243054618205 + ] + } + }, + "fit_power_law": { + "gamma": 1.0097027213131609, + "log_A": -1.6365226792336454, + "R2": 0.980958, + "n_points": 5 + }, + "fit_exponential": { + "lambda": 0.0057384565671822554, + "log_A": -5.358287036922847, + "R2": 0.947259, + "n_points": 5 + }, + "delta_R2_power_minus_exp": 0.0337, + "gamma_CI_95": { + "lo": 0.8032432925460105, + "hi": 1.245778850043926 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/gpt2-xl_random.json b/data/e4_gamma/gpt2-xl_random.json new file mode 100644 index 0000000000000000000000000000000000000000..aaca9b0a12efc73ad8e50c22c5d216c467d528e3 --- /dev/null +++ b/data/e4_gamma/gpt2-xl_random.json @@ -0,0 +1,112 @@ +{ + "model": "gpt2-xl", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.011626105720384255, + "std_across_seeds": 0.00013311837406557365, + "per_seed_means": [ + 0.01144652757793665, + 0.011764828627929092, + 0.01166696095528702 + ] + }, + "20": { + "mean_across_seeds": 0.007488407470906774, + "std_across_seeds": 0.00016811116221478994, + "per_seed_means": [ + 0.0073088953675081334, + 0.0074431687158842885, + 0.007713158329327901 + ] + }, + "30": { + "mean_across_seeds": 0.0058971412112522464, + "std_across_seeds": 0.00022329427885202344, + "per_seed_means": [ + 0.005722023959582051, + 0.0057571235562985145, + 0.0062122761178761724 + ] + }, + "50": { + "mean_across_seeds": 0.003975331396278408, + "std_across_seeds": 4.0324609731129365e-05, + "per_seed_means": [ + 0.0039790187543258075, + 0.004022771728535493, + 0.003924203705973923 + ] + }, + "100": { + "mean_across_seeds": 0.002095724871614948, + "std_across_seeds": 6.976401208546233e-05, + "per_seed_means": [ + 0.0020236784417647868, + 0.0020733742051136993, + 0.002190121967966358 + ] + }, + "200": { + "mean_across_seeds": 0.0010856468626298011, + "std_across_seeds": 1.823042642173508e-05, + "per_seed_means": [ + 0.001111350361413012, + 0.001074533009668812, + 0.001071057216807579 + ] + }, + "500": { + "mean_across_seeds": 0.0003226076814139055, + "std_across_seeds": 1.2754267246726592e-05, + "per_seed_means": [ + 0.0003332430636510253, + 0.0003299063876814519, + 0.0003046735929092392 + ] + } + }, + "fit_power_law": { + "gamma": 1.024099984346949, + "log_A": -1.539486602310909, + "R2": 0.988966, + "n_points": 5 + }, + "fit_exponential": { + "lambda": 0.005748540084502454, + "log_A": -5.326945624997276, + "R2": 0.931594, + "n_points": 5 + }, + "delta_R2_power_minus_exp": 0.0574, + "gamma_CI_95": { + "lo": 0.8592953221206775, + "hi": 1.1981958076311148 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/gpt2_mongo.json b/data/e4_gamma/gpt2_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..9822b777757eb1809d2a3b73680e9a07900e43a1 --- /dev/null +++ b/data/e4_gamma/gpt2_mongo.json @@ -0,0 +1,112 @@ +{ + "model": "gpt2", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.010795217133644555, + "std_across_seeds": 0.00023681001214917195, + "per_seed_means": [ + 0.010487454771064221, + 0.01106346709964176, + 0.01083472953022768 + ] + }, + "20": { + "mean_across_seeds": 0.00582418749988493, + "std_across_seeds": 0.0005571907346203261, + "per_seed_means": [ + 0.005037826988069961, + 0.006173548453953117, + 0.0062611870576317116 + ] + }, + "30": { + "mean_across_seeds": 0.003858918526675552, + "std_across_seeds": 0.00010065529911672517, + "per_seed_means": [ + 0.0039899590720112125, + 0.0037452472886070607, + 0.003841549219408383 + ] + }, + "50": { + "mean_across_seeds": 0.002250000941243747, + "std_across_seeds": 9.438098473233559e-05, + "per_seed_means": [ + 0.002258442856060962, + 0.0021304187859641386, + 0.0023611411817061403 + ] + }, + "100": { + "mean_across_seeds": 0.0014039013699706025, + "std_across_seeds": 0.0001469032439734343, + "per_seed_means": [ + 0.0014018270258869354, + 0.0012250285160068112, + 0.0015848485680180601 + ] + }, + "200": { + "mean_across_seeds": 0.0008754040200276196, + "std_across_seeds": 9.033352540181327e-05, + "per_seed_means": [ + 0.0009984061741852201, + 0.0007840186448690171, + 0.0008437872410286218 + ] + }, + "500": { + "mean_across_seeds": 0.00018121012873557952, + "std_across_seeds": 1.1061475994520298e-05, + "per_seed_means": [ + 0.00018006086377378476, + 0.00016827388525295344, + 0.0001952956371800004 + ] + } + }, + "fit_power_law": { + "gamma": 1.0231174997210821, + "log_A": -1.981276401516759, + "R2": 0.955115, + "n_points": 5 + }, + "fit_exponential": { + "lambda": 0.0059368069236711875, + "log_A": -5.730996278955188, + "R2": 0.961447, + "n_points": 5 + }, + "delta_R2_power_minus_exp": -0.0063, + "gamma_CI_95": { + "lo": 0.680893804842034, + "hi": 1.370306159496448 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/gpt2_random.json b/data/e4_gamma/gpt2_random.json new file mode 100644 index 0000000000000000000000000000000000000000..52763271109e715a85cebf416ba592dd950e5a14 --- /dev/null +++ b/data/e4_gamma/gpt2_random.json @@ -0,0 +1,112 @@ +{ + "model": "gpt2", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.009518861557460493, + "std_across_seeds": 0.00012968562380189386, + "per_seed_means": [ + 0.00936439356766641, + 0.009681724236967663, + 0.009510466867747406 + ] + }, + "20": { + "mean_across_seeds": 0.005003941716471065, + "std_across_seeds": 9.68532116870119e-06, + "per_seed_means": [ + 0.004991514612920582, + 0.005015143603862573, + 0.005005166932630042 + ] + }, + "30": { + "mean_across_seeds": 0.0035997193480013976, + "std_across_seeds": 0.00011978243325751259, + "per_seed_means": [ + 0.0036358602391555906, + 0.0034383236640132962, + 0.003724974140835305 + ] + }, + "50": { + "mean_across_seeds": 0.0021072820673644957, + "std_across_seeds": 4.932407757843146e-05, + "per_seed_means": [ + 0.0020570240674229958, + 0.002090519576255853, + 0.0021743025584146382 + ] + }, + "100": { + "mean_across_seeds": 0.0010727742979846275, + "std_across_seeds": 2.6880493031373388e-05, + "per_seed_means": [ + 0.001041360297628368, + 0.0010699418322959295, + 0.0011070207640295848 + ] + }, + "200": { + "mean_across_seeds": 0.0010083141644200723, + "std_across_seeds": 3.952637175859924e-05, + "per_seed_means": [ + 0.0010641112684970722, + 0.000983333399053663, + 0.000977497825709482 + ] + }, + "500": { + "mean_across_seeds": 0.00014045896414447472, + "std_across_seeds": 1.1344109439322876e-05, + "per_seed_means": [ + 0.00014960293665353675, + 0.00014730293398315553, + 0.0001244710217967319 + ] + } + }, + "fit_power_law": { + "gamma": 1.0514830126336872, + "log_A": -1.9518405504715013, + "R2": 0.907872, + "n_points": 5 + }, + "fit_exponential": { + "lambda": 0.006208089829262706, + "log_A": -5.786742896053141, + "R2": 0.94613, + "n_points": 5 + }, + "delta_R2_power_minus_exp": -0.0383, + "gamma_CI_95": { + "lo": 0.5317191109968791, + "hi": 1.4584073583816823 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/meta-llama--Llama-2-13b-hf_mongo_kaggle.json b/data/e4_gamma/meta-llama--Llama-2-13b-hf_mongo_kaggle.json new file mode 100644 index 0000000000000000000000000000000000000000..afaa35924dbe948476fc9b816da8fe11ec35d5ff --- /dev/null +++ b/data/e4_gamma/meta-llama--Llama-2-13b-hf_mongo_kaggle.json @@ -0,0 +1,32 @@ +{ + "model": "meta-llama/Llama-2-13b-hf", + "corpus": "mongo_sim", + "platform": "Kaggle T4 x2 Linux", + "note": "fit d>=50 gamma=0.996 R2=1.000 Delta_gamma=-0.003", + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.1814 + }, + "20": { + "mean_across_seeds": 0.113976 + }, + "30": { + "mean_across_seeds": 0.065681 + }, + "50": { + "mean_across_seeds": 0.028472 + }, + "100": { + "mean_across_seeds": 0.014255 + }, + "200": { + "mean_across_seeds": 0.007158 + }, + "500": { + "mean_across_seeds": 0.002896 + }, + "1000": { + "mean_across_seeds": 0.001441 + } + } +} \ No newline at end of file diff --git a/data/e4_gamma/meta-llama--Llama-2-13b-hf_random_kaggle.json b/data/e4_gamma/meta-llama--Llama-2-13b-hf_random_kaggle.json new file mode 100644 index 0000000000000000000000000000000000000000..d126684bd448b3ad5b1b6f6109ba9ff28318bda8 --- /dev/null +++ b/data/e4_gamma/meta-llama--Llama-2-13b-hf_random_kaggle.json @@ -0,0 +1,32 @@ +{ + "model": "meta-llama/Llama-2-13b-hf", + "corpus": "random", + "platform": "Kaggle T4 x2 Linux", + "note": "fit d>=50 gamma=0.999 R2=0.9998", + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.179776 + }, + "20": { + "mean_across_seeds": 0.087794 + }, + "30": { + "mean_across_seeds": 0.057804 + }, + "50": { + "mean_across_seeds": 0.034621 + }, + "100": { + "mean_across_seeds": 0.017573 + }, + "200": { + "mean_across_seeds": 0.008675 + }, + "500": { + "mean_across_seeds": 0.003262 + }, + "1000": { + "mean_across_seeds": 0.0016 + } + } +} \ No newline at end of file diff --git a/data/e4_gamma/meta-llama--Llama-2-7b-hf_mongo.bak.json b/data/e4_gamma/meta-llama--Llama-2-7b-hf_mongo.bak.json new file mode 100644 index 0000000000000000000000000000000000000000..931a949073d7b7300bb8d5635d3304c0f6a25360 --- /dev/null +++ b/data/e4_gamma/meta-llama--Llama-2-7b-hf_mongo.bak.json @@ -0,0 +1,134 @@ +{ + "model": "meta-llama/Llama-2-7b-hf", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.005839068323192705, + "std_across_seeds": 0.00100414805142047, + "per_seed_means": [ + 0.005180615605010341, + 0.0072579276375472545, + 0.005078661727020517 + ] + }, + "20": { + "mean_across_seeds": 0.003595085684986164, + "std_across_seeds": 0.0009065959864917142, + "per_seed_means": [ + 0.0030458338442258537, + 0.004873014269396663, + 0.002866408941335976 + ] + }, + "30": { + "mean_across_seeds": 0.0029745564295444635, + "std_across_seeds": 0.0011814371368035767, + "per_seed_means": [ + 0.002172227150682981, + 0.004644930018888166, + 0.002106512119062245 + ] + }, + "50": { + "mean_across_seeds": 0.00218253040427549, + "std_across_seeds": 0.0009243277252711826, + "per_seed_means": [ + 0.0014683960599359124, + 0.003487796774522091, + 0.0015913983783684671 + ] + }, + "100": { + "mean_across_seeds": 0.0015408689192392758, + "std_across_seeds": 0.0009065662521180087, + "per_seed_means": [ + 0.0009471153894749781, + 0.0028218117965540536, + 0.0008536795716887961 + ] + }, + "200": { + "mean_across_seeds": 0.0009803178196307273, + "std_across_seeds": 0.0007938919491388493, + "per_seed_means": [ + 0.00044963144852469366, + 0.002102501493257781, + 0.000388820517109707 + ] + }, + "500": { + "mean_across_seeds": 0.0008679315625017303, + "std_across_seeds": 0.0009970067991655464, + "per_seed_means": [ + 0.0001939495940071841, + 0.002277463952001805, + 0.00013238114149620136 + ] + }, + "1000": { + "mean_across_seeds": 0.0009244536463585166, + "std_across_seeds": 0.0011925700211172295, + "per_seed_means": [ + 9.362193833415707e-05, + 0.002610941444678853, + 6.879755606253942e-05 + ] + }, + "2000": { + "mean_across_seeds": 0.0008947346011538887, + "std_across_seeds": 0.0011917730902519723, + "per_seed_means": [ + 6.526854238472879e-05, + 0.002580087239621207, + 3.884802145573e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.2870574377368437, + "log_A": -5.073101472705059, + "R2": 0.814928, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0004258480767616808, + "log_A": -6.392910204458021, + "R2": 0.386505, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.4284, + "gamma_CI_95": { + "lo": 0.12877777836886914, + "hi": 0.5472656629105128 + }, + "decision": "UNCLEAR: \u03b3=0.287 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/meta-llama--Llama-2-7b-hf_mongo.json b/data/e4_gamma/meta-llama--Llama-2-7b-hf_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..931a949073d7b7300bb8d5635d3304c0f6a25360 --- /dev/null +++ b/data/e4_gamma/meta-llama--Llama-2-7b-hf_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "meta-llama/Llama-2-7b-hf", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.005839068323192705, + "std_across_seeds": 0.00100414805142047, + "per_seed_means": [ + 0.005180615605010341, + 0.0072579276375472545, + 0.005078661727020517 + ] + }, + "20": { + "mean_across_seeds": 0.003595085684986164, + "std_across_seeds": 0.0009065959864917142, + "per_seed_means": [ + 0.0030458338442258537, + 0.004873014269396663, + 0.002866408941335976 + ] + }, + "30": { + "mean_across_seeds": 0.0029745564295444635, + "std_across_seeds": 0.0011814371368035767, + "per_seed_means": [ + 0.002172227150682981, + 0.004644930018888166, + 0.002106512119062245 + ] + }, + "50": { + "mean_across_seeds": 0.00218253040427549, + "std_across_seeds": 0.0009243277252711826, + "per_seed_means": [ + 0.0014683960599359124, + 0.003487796774522091, + 0.0015913983783684671 + ] + }, + "100": { + "mean_across_seeds": 0.0015408689192392758, + "std_across_seeds": 0.0009065662521180087, + "per_seed_means": [ + 0.0009471153894749781, + 0.0028218117965540536, + 0.0008536795716887961 + ] + }, + "200": { + "mean_across_seeds": 0.0009803178196307273, + "std_across_seeds": 0.0007938919491388493, + "per_seed_means": [ + 0.00044963144852469366, + 0.002102501493257781, + 0.000388820517109707 + ] + }, + "500": { + "mean_across_seeds": 0.0008679315625017303, + "std_across_seeds": 0.0009970067991655464, + "per_seed_means": [ + 0.0001939495940071841, + 0.002277463952001805, + 0.00013238114149620136 + ] + }, + "1000": { + "mean_across_seeds": 0.0009244536463585166, + "std_across_seeds": 0.0011925700211172295, + "per_seed_means": [ + 9.362193833415707e-05, + 0.002610941444678853, + 6.879755606253942e-05 + ] + }, + "2000": { + "mean_across_seeds": 0.0008947346011538887, + "std_across_seeds": 0.0011917730902519723, + "per_seed_means": [ + 6.526854238472879e-05, + 0.002580087239621207, + 3.884802145573e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.2870574377368437, + "log_A": -5.073101472705059, + "R2": 0.814928, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0004258480767616808, + "log_A": -6.392910204458021, + "R2": 0.386505, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.4284, + "gamma_CI_95": { + "lo": 0.12877777836886914, + "hi": 0.5472656629105128 + }, + "decision": "UNCLEAR: \u03b3=0.287 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/meta-llama--Llama-2-7b-hf_mongo_kaggle.json b/data/e4_gamma/meta-llama--Llama-2-7b-hf_mongo_kaggle.json new file mode 100644 index 0000000000000000000000000000000000000000..88b29efbfce4c01ba7b114ee249c251b8be604f9 --- /dev/null +++ b/data/e4_gamma/meta-llama--Llama-2-7b-hf_mongo_kaggle.json @@ -0,0 +1,32 @@ +{ + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "mongo_sim", + "platform": "Kaggle T4 x2 Linux", + "note": "fit d>=50 gamma=1.0737 R2=0.9969", + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.177255 + }, + "20": { + "mean_across_seeds": 0.086428 + }, + "30": { + "mean_across_seeds": 0.054991 + }, + "50": { + "mean_across_seeds": 0.034669 + }, + "100": { + "mean_across_seeds": 0.016608 + }, + "200": { + "mean_across_seeds": 0.008696 + }, + "500": { + "mean_across_seeds": 0.001737 + }, + "1000": { + "mean_across_seeds": 0.000899 + } + } +} \ No newline at end of file diff --git a/data/e4_gamma/meta-llama--Llama-2-7b-hf_random.json b/data/e4_gamma/meta-llama--Llama-2-7b-hf_random.json new file mode 100644 index 0000000000000000000000000000000000000000..c5ea1d1b71497ffe08c07e969934ec08e3c081a1 --- /dev/null +++ b/data/e4_gamma/meta-llama--Llama-2-7b-hf_random.json @@ -0,0 +1,134 @@ +{ + "model": "meta-llama/Llama-2-7b-hf", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.005234783389605581, + "std_across_seeds": 0.0001605784847157718, + "per_seed_means": [ + 0.005241896160878241, + 0.005427798189533254, + 0.0050346558184052505 + ] + }, + "20": { + "mean_across_seeds": 0.0032586656397001613, + "std_across_seeds": 9.004189968687237e-05, + "per_seed_means": [ + 0.0033254111697897317, + 0.0033192082153012353, + 0.003131377534009516 + ] + }, + "30": { + "mean_across_seeds": 0.0027830680216559107, + "std_across_seeds": 7.708031161773844e-05, + "per_seed_means": [ + 0.0027384231130902965, + 0.002891513560898602, + 0.002719267390978833 + ] + }, + "50": { + "mean_across_seeds": 0.0020982515016415466, + "std_across_seeds": 1.0997527870222363e-05, + "per_seed_means": [ + 0.0020832987447890142, + 0.0021094332445257655, + 0.0021020225156098606 + ] + }, + "100": { + "mean_across_seeds": 0.0013744194064444551, + "std_across_seeds": 0.0001405009107521968, + "per_seed_means": [ + 0.0012892550001076112, + 0.0015724719502031802, + 0.001261531269022574 + ] + }, + "200": { + "mean_across_seeds": 0.0006745984775221183, + "std_across_seeds": 2.1830080952722776e-05, + "per_seed_means": [ + 0.0006595045716191332, + 0.0006588224985171109, + 0.0007054683624301106 + ] + }, + "500": { + "mean_across_seeds": 0.0003505304501029766, + "std_across_seeds": 8.78040776303756e-05, + "per_seed_means": [ + 0.00027183269693826633, + 0.0004730618477333337, + 0.0003066968056373298 + ] + }, + "1000": { + "mean_across_seeds": 0.00016617169836536053, + "std_across_seeds": 4.34756847288954e-05, + "per_seed_means": [ + 0.00014104328583925964, + 0.00022733245704633495, + 0.00013013935221048694 + ] + }, + "2000": { + "mean_across_seeds": 9.237132345636685e-05, + "std_across_seeds": 1.932525645683395e-06, + "per_seed_means": [ + 9.182618930935859e-05, + 9.496318021168312e-05, + 9.032460084805886e-05 + ] + } + }, + "fit_power_law": { + "gamma": 0.8266242679750889, + "log_A": -2.9325874169558817, + "R2": 0.993628, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0016182629237530125, + "log_A": -6.515907832083295, + "R2": 0.820672, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.173, + "gamma_CI_95": { + "lo": 0.744966636210376, + "hi": 0.902244750583129 + }, + "decision": "UNCLEAR: \u03b3=0.827 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/meta-llama--Llama-2-7b-hf_random_kaggle.json b/data/e4_gamma/meta-llama--Llama-2-7b-hf_random_kaggle.json new file mode 100644 index 0000000000000000000000000000000000000000..005be17b2e70a22a552cb96e8b11741270688510 --- /dev/null +++ b/data/e4_gamma/meta-llama--Llama-2-7b-hf_random_kaggle.json @@ -0,0 +1,32 @@ +{ + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "random", + "platform": "Kaggle T4 x2 Linux", + "note": "fit d>=50 gamma=1.005 R2=0.9999", + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.176144 + }, + "20": { + "mean_across_seeds": 0.085621 + }, + "30": { + "mean_across_seeds": 0.056189 + }, + "50": { + "mean_across_seeds": 0.03359 + }, + "100": { + "mean_across_seeds": 0.016926 + }, + "200": { + "mean_across_seeds": 0.008361 + }, + "500": { + "mean_across_seeds": 0.003127 + }, + "1000": { + "mean_across_seeds": 0.001554 + } + } +} \ No newline at end of file diff --git a/data/e4_gamma/meta-llama--Meta-Llama-3-8B_mongo.json b/data/e4_gamma/meta-llama--Meta-Llama-3-8B_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..caa8ada86f6dbf25904fb959dc020bf43d87d735 --- /dev/null +++ b/data/e4_gamma/meta-llama--Meta-Llama-3-8B_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "meta-llama/Meta-Llama-3-8B", + "theta": 500000, + "gamma_pred": 0.7018781400200674, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.0059040391059695845, + "std_across_seeds": 7.517924513277696e-05, + "per_seed_means": [ + 0.005812099125857155, + 0.0059037688712123785, + 0.005996249320839221 + ] + }, + "20": { + "mean_across_seeds": 0.0036818826782594742, + "std_across_seeds": 9.199067084087526e-05, + "per_seed_means": [ + 0.003595458084406952, + 0.0036408838270775355, + 0.0038093061232939364 + ] + }, + "30": { + "mean_across_seeds": 0.0025728944933184213, + "std_across_seeds": 0.0002863462536592025, + "per_seed_means": [ + 0.0021755760831486744, + 0.002839338845612171, + 0.0027037685511944196 + ] + }, + "50": { + "mean_across_seeds": 0.0014714971260077114, + "std_across_seeds": 0.00015744269355329684, + "per_seed_means": [ + 0.001322247434873134, + 0.0016892152974226822, + 0.0014030286457273177 + ] + }, + "100": { + "mean_across_seeds": 0.0006953646930115712, + "std_across_seeds": 4.0588058334478556e-05, + "per_seed_means": [ + 0.0006596940151939634, + 0.0007521459916218495, + 0.0006742540722189006 + ] + }, + "200": { + "mean_across_seeds": 0.00037121372909395075, + "std_across_seeds": 1.2735057802721339e-05, + "per_seed_means": [ + 0.00035418200811060765, + 0.0003848003790820561, + 0.0003746588000891885 + ] + }, + "500": { + "mean_across_seeds": 0.00011184491504738818, + "std_across_seeds": 2.5384733108484725e-05, + "per_seed_means": [ + 9.096407898444643e-05, + 9.699561132341236e-05, + 0.00014757505483430577 + ] + }, + "1000": { + "mean_across_seeds": 6.471107897798195e-05, + "std_across_seeds": 6.908128572735838e-06, + "per_seed_means": [ + 5.5370496123335516e-05, + 7.186098620574437e-05, + 6.690175460486595e-05 + ] + }, + "2000": { + "mean_across_seeds": 3.34952792859945e-05, + "std_across_seeds": 1.2209969790636401e-06, + "per_seed_means": [ + 3.2678065360111455e-05, + 3.2586551255917585e-05, + 3.522122124195448e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.0454762537473639, + "log_A": -2.4338207488763257, + "R2": 0.997461, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.001961969550831471, + "log_A": -7.012806433730394, + "R2": 0.757035, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2404, + "gamma_CI_95": { + "lo": 1.0179726763287769, + "hi": 1.10705984349428 + }, + "decision": "REFUTED: C not constant across \u03b8" +} \ No newline at end of file diff --git a/data/e4_gamma/meta-llama--Meta-Llama-3-8B_random.json b/data/e4_gamma/meta-llama--Meta-Llama-3-8B_random.json new file mode 100644 index 0000000000000000000000000000000000000000..c4e059199b5271506207c1fa2e210ebb9f52d70c --- /dev/null +++ b/data/e4_gamma/meta-llama--Meta-Llama-3-8B_random.json @@ -0,0 +1,134 @@ +{ + "model": "meta-llama/Meta-Llama-3-8B", + "theta": 500000, + "gamma_pred": 0.7018781400200674, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.006357483927212242, + "std_across_seeds": 0.00016796054505855608, + "per_seed_means": [ + 0.006397274050395935, + 0.006134786802964906, + 0.006540390928275883 + ] + }, + "20": { + "mean_across_seeds": 0.004401154224372779, + "std_across_seeds": 0.0002582280036923812, + "per_seed_means": [ + 0.004376536819230144, + 0.004097918888243536, + 0.004729006965644658 + ] + }, + "30": { + "mean_across_seeds": 0.0032561490674399667, + "std_across_seeds": 0.00016113210865905016, + "per_seed_means": [ + 0.003459152732199679, + 0.003064995304060479, + 0.0032442991660597423 + ] + }, + "50": { + "mean_across_seeds": 0.002258875768683437, + "std_across_seeds": 0.00013491633461167046, + "per_seed_means": [ + 0.0021290461854853978, + 0.002202704493732502, + 0.0024448766268324105 + ] + }, + "100": { + "mean_across_seeds": 0.001293239965258787, + "std_across_seeds": 6.179488791374727e-05, + "per_seed_means": [ + 0.0012110524675032744, + 0.0013086077803745866, + 0.0013600596478985 + ] + }, + "200": { + "mean_across_seeds": 0.0007211868295497778, + "std_across_seeds": 1.3865975313109934e-05, + "per_seed_means": [ + 0.0007325983955524862, + 0.0007292915508151054, + 0.0007016705422817419 + ] + }, + "500": { + "mean_across_seeds": 0.0002737677599199944, + "std_across_seeds": 7.60590967347278e-05, + "per_seed_means": [ + 0.00038133149850182234, + 0.00022009029790448647, + 0.00021988148335367442 + ] + }, + "1000": { + "mean_across_seeds": 0.0002607676472204427, + "std_across_seeds": 5.930906827936715e-05, + "per_seed_means": [ + 0.00017891603560807803, + 0.00028583206507998206, + 0.00031755484097326797 + ] + }, + "2000": { + "mean_across_seeds": 0.00013976724688998527, + "std_across_seeds": 4.561438863275292e-05, + "per_seed_means": [ + 0.000117663445804889, + 9.833510072591405e-05, + 0.00020330319413915275 + ] + } + }, + "fit_power_law": { + "gamma": 0.7589145044527899, + "log_A": -3.177537408176203, + "R2": 0.984284, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.0013970251968084694, + "log_A": -6.516500070173739, + "R2": 0.718799, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2655, + "gamma_CI_95": { + "lo": 0.7038410605464432, + "hi": 0.8918247508895275 + }, + "decision": "CONFIRMED: \u03b3 law holds (\u03b3\u00d7ln(\u03b8) = C)" +} \ No newline at end of file diff --git a/data/e4_gamma/microsoft--phi-2_mongo.json b/data/e4_gamma/microsoft--phi-2_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..57a8277d2531fa38d2bf7a9a4a89126ddfc09e7e --- /dev/null +++ b/data/e4_gamma/microsoft--phi-2_mongo.json @@ -0,0 +1,123 @@ +{ + "model": "microsoft/phi-2", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 30, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.00599521297780383, + "std_across_seeds": 0.0006854533297223726, + "per_seed_means": [ + 0.00520636859194686, + 0.006877551968985548, + 0.005901718372479081 + ] + }, + "20": { + "mean_across_seeds": 0.004053244446145577, + "std_across_seeds": 0.0008138386774588067, + "per_seed_means": [ + 0.003397960588335991, + 0.005200308359538515, + 0.0035614643905622265 + ] + }, + "30": { + "mean_across_seeds": 0.0031503823217159757, + "std_across_seeds": 0.0006758948028724937, + "per_seed_means": [ + 0.0022145009386197972, + 0.003786683354216317, + 0.0034499626723118126 + ] + }, + "50": { + "mean_across_seeds": 0.0018561340684148794, + "std_across_seeds": 0.0005290413098087929, + "per_seed_means": [ + 0.0013084483109802628, + 0.002571403378775964, + 0.0016885505154884109 + ] + }, + "100": { + "mean_across_seeds": 0.0010692747583056592, + "std_across_seeds": 0.000252071604127596, + "per_seed_means": [ + 0.0007919824856799096, + 0.0014019360668802014, + 0.001013905722356867 + ] + }, + "200": { + "mean_across_seeds": 0.0006019849353631596, + "std_across_seeds": 0.00023232540169545936, + "per_seed_means": [ + 0.00038718529782878856, + 0.00092469493780906, + 0.0004940745704516303 + ] + }, + "500": { + "mean_across_seeds": 0.00012888134598243697, + "std_across_seeds": 4.62531287050378e-06, + "per_seed_means": [ + 0.0001247542658650976, + 0.00012654992460738867, + 0.00013533984747482464 + ] + }, + "1000": { + "mean_across_seeds": 9.522781525043278e-05, + "std_across_seeds": 1.4165200908451387e-05, + "per_seed_means": [ + 0.0001097464019646092, + 7.60149720008485e-05, + 9.992207178584067e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.0446992618601783, + "log_A": -2.1376991856798906, + "R2": 0.979964, + "n_points": 6 + }, + "fit_exponential": { + "lambda": 0.003418448687958094, + "log_A": -6.349119498523909, + "R2": 0.825434, + "n_points": 6 + }, + "delta_R2_power_minus_exp": 0.1545, + "gamma_CI_95": { + "lo": 0.8542023295812078, + "hi": 1.202500232353316 + }, + "decision": "CONFIRMED: \u03b3 law holds (\u03b3\u00d7ln(\u03b8) = C)" +} \ No newline at end of file diff --git a/data/e4_gamma/microsoft--phi-2_random.json b/data/e4_gamma/microsoft--phi-2_random.json new file mode 100644 index 0000000000000000000000000000000000000000..bc38b534aebadecf5286219ec3153c8fa6c2cb37 --- /dev/null +++ b/data/e4_gamma/microsoft--phi-2_random.json @@ -0,0 +1,123 @@ +{ + "model": "microsoft/phi-2", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 30, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.007808198016654287, + "std_across_seeds": 0.0005307177778644769, + "per_seed_means": [ + 0.007057766010984778, + 0.008171973160157602, + 0.00819485487882048 + ] + }, + "20": { + "mean_across_seeds": 0.00550748185099413, + "std_across_seeds": 0.00011964781342648696, + "per_seed_means": [ + 0.005383339112934967, + 0.005669126481128235, + 0.005469979958919188 + ] + }, + "30": { + "mean_across_seeds": 0.0035577644354715526, + "std_across_seeds": 0.00012055662534639748, + "per_seed_means": [ + 0.0033873209768595794, + 0.0036394336396673075, + 0.003646538689887772 + ] + }, + "50": { + "mean_across_seeds": 0.0022924808707709115, + "std_across_seeds": 7.453685612076765e-05, + "per_seed_means": [ + 0.002285287519528841, + 0.0022050017199944706, + 0.0023871533727894225 + ] + }, + "100": { + "mean_across_seeds": 0.002331886454744058, + "std_across_seeds": 8.946231158808722e-05, + "per_seed_means": [ + 0.002344710259543111, + 0.002434478773890684, + 0.0022164703307983777 + ] + }, + "200": { + "mean_across_seeds": 0.0010912099793333456, + "std_across_seeds": 5.543814799433026e-05, + "per_seed_means": [ + 0.001096232064689199, + 0.0011564570828340947, + 0.0010209407904767432 + ] + }, + "500": { + "mean_across_seeds": 0.00031415204034096353, + "std_across_seeds": 4.151945435990568e-05, + "per_seed_means": [ + 0.00029279896795439224, + 0.00027745946887686537, + 0.00037219768419163304 + ] + }, + "1000": { + "mean_across_seeds": 0.0001819735528745999, + "std_across_seeds": 4.337414477393199e-05, + "per_seed_means": [ + 0.00024319090977466354, + 0.00014800316906378916, + 0.00015472657978534698 + ] + } + }, + "fit_power_law": { + "gamma": 0.8707349253167322, + "log_A": -2.476380517784226, + "R2": 0.947956, + "n_points": 6 + }, + "fit_exponential": { + "lambda": 0.0030164061531184564, + "log_A": -5.934121422709407, + "R2": 0.894936, + "n_points": 6 + }, + "delta_R2_power_minus_exp": 0.053, + "gamma_CI_95": { + "lo": 0.5198722145183583, + "hi": 1.168125861223296 + }, + "decision": "UNCLEAR: \u03b3=0.871 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/microsoft--phi-3-mini-4k-instruct_mongo.json b/data/e4_gamma/microsoft--phi-3-mini-4k-instruct_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..ba5c09588092286466808470fa3d42cd39cd8519 --- /dev/null +++ b/data/e4_gamma/microsoft--phi-3-mini-4k-instruct_mongo.json @@ -0,0 +1,123 @@ +{ + "model": "microsoft/phi-3-mini-4k-instruct", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 30, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.007768188691827365, + "std_across_seeds": 0.0007225399094016229, + "per_seed_means": [ + 0.00691897339032342, + 0.008684955777910848, + 0.007700636907247827 + ] + }, + "20": { + "mean_across_seeds": 0.0047616929113347495, + "std_across_seeds": 0.0012876405883475589, + "per_seed_means": [ + 0.0033099546291244526, + 0.006439596399043997, + 0.0045355277058358 + ] + }, + "30": { + "mean_across_seeds": 0.0034923262092181376, + "std_across_seeds": 0.0008340820263114457, + "per_seed_means": [ + 0.0027606055703169358, + 0.004659421774946774, + 0.0030569512823907037 + ] + }, + "50": { + "mean_across_seeds": 0.002236078068381175, + "std_across_seeds": 0.0007137522912032439, + "per_seed_means": [ + 0.001717447629198432, + 0.0032453468535095452, + 0.001745439722435549 + ] + }, + "100": { + "mean_across_seeds": 0.0015452678043705722, + "std_across_seeds": 0.0006619245783128043, + "per_seed_means": [ + 0.000911949206298838, + 0.0024589169576453664, + 0.0012649372491675118 + ] + }, + "200": { + "mean_across_seeds": 0.0009376381168193702, + "std_across_seeds": 0.0006441538549782175, + "per_seed_means": [ + 0.0003973938951579233, + 0.0018429794174153358, + 0.0005725410378848513 + ] + }, + "500": { + "mean_across_seeds": 0.0004735038357062472, + "std_across_seeds": 0.00044506573421803784, + "per_seed_means": [ + 0.00010263245397557815, + 0.0010993553247923653, + 0.0002185237283507983 + ] + }, + "1000": { + "mean_across_seeds": 0.0004106088735473653, + "std_across_seeds": 0.0004407722295620771, + "per_seed_means": [ + 7.967643129328887e-05, + 0.0010335497131260733, + 0.00011860047622273366 + ] + } + }, + "fit_power_law": { + "gamma": 0.6295631683122206, + "log_A": -3.5929636163535084, + "R2": 0.985314, + "n_points": 6 + }, + "fit_exponential": { + "lambda": 0.0019634326299251487, + "log_A": -6.161148369310343, + "R2": 0.753919, + "n_points": 6 + }, + "delta_R2_power_minus_exp": 0.2314, + "gamma_CI_95": { + "lo": 0.563087100915038, + "hi": 0.7114507478893123 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/microsoft--phi-3-mini-4k-instruct_random.json b/data/e4_gamma/microsoft--phi-3-mini-4k-instruct_random.json new file mode 100644 index 0000000000000000000000000000000000000000..d4f3c131e9b078a514253f1aa58ccb2bc875c182 --- /dev/null +++ b/data/e4_gamma/microsoft--phi-3-mini-4k-instruct_random.json @@ -0,0 +1,123 @@ +{ + "model": "microsoft/phi-3-mini-4k-instruct", + "theta": null, + "gamma_pred": null, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.006656107790250745, + "std_across_seeds": 0.000161820624324973, + "per_seed_means": [ + 0.006711039271516104, + 0.00682103688052545, + 0.006436247218710681 + ] + }, + "20": { + "mean_across_seeds": 0.004706236780879812, + "std_across_seeds": 0.00013624776591829505, + "per_seed_means": [ + 0.004527395272937914, + 0.0048577620182186365, + 0.004733553051482886 + ] + }, + "30": { + "mean_across_seeds": 0.0035314693899514775, + "std_across_seeds": 8.65089860543182e-05, + "per_seed_means": [ + 0.003650119398565342, + 0.0034979772831623753, + 0.003446311488126715 + ] + }, + "50": { + "mean_across_seeds": 0.002377997318205113, + "std_across_seeds": 4.332318604246771e-05, + "per_seed_means": [ + 0.0023252750479150564, + 0.002377328482301285, + 0.0024313884243989986 + ] + }, + "100": { + "mean_across_seeds": 0.0016664358042180539, + "std_across_seeds": 2.4892496387242913e-05, + "per_seed_means": [ + 0.0016318851887869338, + 0.0016778676716300348, + 0.0016895545522371928 + ] + }, + "200": { + "mean_across_seeds": 0.0007248134792058004, + "std_across_seeds": 1.5025375955684408e-05, + "per_seed_means": [ + 0.0007054781843908131, + 0.000742113469944646, + 0.0007268487832819422 + ] + }, + "500": { + "mean_across_seeds": 0.0001851141119065384, + "std_across_seeds": 1.835492373684119e-06, + "per_seed_means": [ + 0.00018557568507579465, + 0.00018267114064656197, + 0.0001870955099972586 + ] + }, + "1000": { + "mean_across_seeds": 0.00011090571746333605, + "std_across_seeds": 3.634582349288755e-06, + "per_seed_means": [ + 0.00011604562828627726, + 0.00010830011335201562, + 0.0001083714107517153 + ] + } + }, + "fit_power_law": { + "gamma": 1.0366024777115062, + "log_A": -1.9276689911542253, + "R2": 0.977923, + "n_points": 6 + }, + "fit_exponential": { + "lambda": 0.00347592517240831, + "log_A": -6.08013852024965, + "R2": 0.865002, + "n_points": 6 + }, + "delta_R2_power_minus_exp": 0.1129, + "gamma_CI_95": { + "lo": 0.771255240048162, + "hi": 1.2524990696604243 + }, + "decision": "unknown" +} \ No newline at end of file diff --git a/data/e4_gamma/mistralai--Mistral-7B-v0.1_mongo.json b/data/e4_gamma/mistralai--Mistral-7B-v0.1_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..6184342c06a372815e77d3869dce5c07f313473f --- /dev/null +++ b/data/e4_gamma/mistralai--Mistral-7B-v0.1_mongo.json @@ -0,0 +1,134 @@ +{ + "model": "mistralai/Mistral-7B-v0.1", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.006733735190031843, + "std_across_seeds": 0.00018158692211685278, + "per_seed_means": [ + 0.006765159337082878, + 0.006938749434969698, + 0.006497296798042953 + ] + }, + "20": { + "mean_across_seeds": 0.004026857788234742, + "std_across_seeds": 0.00024782820850164717, + "per_seed_means": [ + 0.004295296672886859, + 0.0036974880056610954, + 0.004087788686156273 + ] + }, + "30": { + "mean_across_seeds": 0.0030662473954726015, + "std_across_seeds": 0.00020010451736387442, + "per_seed_means": [ + 0.0031750365230254827, + 0.00278560866912206, + 0.00323809699427026 + ] + }, + "50": { + "mean_across_seeds": 0.0019213843246042315, + "std_across_seeds": 9.756523461466468e-05, + "per_seed_means": [ + 0.002050339955991755, + 0.0018144059635233135, + 0.001899407054297626 + ] + }, + "100": { + "mean_across_seeds": 0.0009038505519533323, + "std_across_seeds": 4.178189010079769e-05, + "per_seed_means": [ + 0.0009573239588644356, + 0.0008988862065598368, + 0.0008553414904357245 + ] + }, + "200": { + "mean_across_seeds": 0.00039353982273799675, + "std_across_seeds": 7.880145375291851e-06, + "per_seed_means": [ + 0.00038251106821311017, + 0.0004004398287118723, + 0.00039766857128900786 + ] + }, + "500": { + "mean_across_seeds": 0.0001694695357905908, + "std_across_seeds": 2.192352943423871e-05, + "per_seed_means": [ + 0.00020012385793961586, + 0.00015011793506952624, + 0.00015816681436263026 + ] + }, + "1000": { + "mean_across_seeds": 8.260491097138987e-05, + "std_across_seeds": 5.810246880950613e-06, + "per_seed_means": [ + 7.441146726099154e-05, + 8.723927894607186e-05, + 8.616398670710623e-05 + ] + }, + "2000": { + "mean_across_seeds": 3.443458402115438e-05, + "std_across_seeds": 2.35840641500017e-06, + "per_seed_means": [ + 3.467809059657157e-05, + 3.143209304350118e-05, + 3.719356842339039e-05 + ] + } + }, + "fit_power_law": { + "gamma": 1.060750419523944, + "log_A": -2.143867119472637, + "R2": 0.99869, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.002038117044881117, + "log_A": -6.763431257485258, + "R2": 0.79456, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2041, + "gamma_CI_95": { + "lo": 1.0287296895523124, + "hi": 1.0879595935791004 + }, + "decision": "CONFIRMED: \u03b3 law holds (\u03b3\u00d7ln(\u03b8) = C)" +} \ No newline at end of file diff --git a/data/e4_gamma/mistralai--Mistral-7B-v0.1_random.json b/data/e4_gamma/mistralai--Mistral-7B-v0.1_random.json new file mode 100644 index 0000000000000000000000000000000000000000..3f1aa284aab8039fb3eac7bbba4c5118c9e5e93a --- /dev/null +++ b/data/e4_gamma/mistralai--Mistral-7B-v0.1_random.json @@ -0,0 +1,134 @@ +{ + "model": "mistralai/Mistral-7B-v0.1", + "theta": 10000, + "gamma_pred": 0.99999561666838, + "C_theory": 9.2103, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "distances_fit": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "d_min_fit": 30, + "n_prompts_per_distance": 150, + "seeds": [ + 42, + 123, + 7 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.009985529906633829, + "std_across_seeds": 0.0020529093554592093, + "per_seed_means": [ + 0.008640429532776276, + 0.008429923014094433, + 0.012886237173030774 + ] + }, + "20": { + "mean_across_seeds": 0.0060750810507064065, + "std_across_seeds": 0.00012791790137236543, + "per_seed_means": [ + 0.0059629041111717625, + 0.0062540789786726236, + 0.006008260062274833 + ] + }, + "30": { + "mean_across_seeds": 0.005252508784696045, + "std_across_seeds": 0.0006080682992417329, + "per_seed_means": [ + 0.004913066189425687, + 0.0047379752846124275, + 0.006106484880050023 + ] + }, + "50": { + "mean_across_seeds": 0.0036803160721643104, + "std_across_seeds": 2.9324149519756347e-05, + "per_seed_means": [ + 0.0037202316313050686, + 0.0036506156173224252, + 0.0036701009678654375 + ] + }, + "100": { + "mean_across_seeds": 0.0022635483171325175, + "std_across_seeds": 6.327280155400579e-05, + "per_seed_means": [ + 0.0023529581003822386, + 0.002221940055799981, + 0.0022157467952153335 + ] + }, + "200": { + "mean_across_seeds": 0.0010285094767136292, + "std_across_seeds": 1.8084447701874e-05, + "per_seed_means": [ + 0.0010491847825081398, + 0.0010051345821314801, + 0.0010312090655012678 + ] + }, + "500": { + "mean_across_seeds": 0.0005741028257438707, + "std_across_seeds": 7.518165123944335e-06, + "per_seed_means": [ + 0.0005697852384764701, + 0.00056784716512387, + 0.0005846760736312718 + ] + }, + "1000": { + "mean_across_seeds": 0.00029642416384174594, + "std_across_seeds": 7.376005709388785e-06, + "per_seed_means": [ + 0.0003041105306086441, + 0.0002864737703930587, + 0.000298688190523535 + ] + }, + "2000": { + "mean_across_seeds": 0.0001669441466219723, + "std_across_seeds": 4.494672754940005e-06, + "per_seed_means": [ + 0.0001644361581808577, + 0.00016313991781013708, + 0.0001732563638749222 + ] + } + }, + "fit_power_law": { + "gamma": 0.8296009929924347, + "log_A": -2.376214984270495, + "R2": 0.996923, + "n_points": 7 + }, + "fit_exponential": { + "lambda": 0.00158832128061495, + "log_A": -5.9922654628941725, + "R2": 0.787524, + "n_points": 7 + }, + "delta_R2_power_minus_exp": 0.2094, + "gamma_CI_95": { + "lo": 0.7916914106876054, + "hi": 0.8739306994591721 + }, + "decision": "UNCLEAR: \u03b3=0.830 outside all expected ranges" +} \ No newline at end of file diff --git a/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_mongo.json b/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..1a5e21db94b6eeb4e3240bff4333f7759b8d4fad --- /dev/null +++ b/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_mongo.json @@ -0,0 +1,90 @@ +{ + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "theta": 1000000, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.008414634897166656, + "std_across_seeds": 0.001494959703850088, + "per_seed_means": [ + 0.0070227079143902905, + 0.010488735257725541, + 0.007732461519384136 + ] + }, + "20": { + "mean_across_seeds": 0.005654149828866746, + "std_across_seeds": 0.001134504996687596, + "per_seed_means": [ + 0.004655646131141111, + 0.007241011513785148, + 0.005065791841673975 + ] + }, + "30": { + "mean_across_seeds": 0.0038466191224546895, + "std_across_seeds": 0.0010558909426180547, + "per_seed_means": [ + 0.0030183024186408148, + 0.005336777986764597, + 0.0031847769619586567 + ] + }, + "50": { + "mean_across_seeds": 0.002776305962017634, + "std_across_seeds": 0.0010345564495630884, + "per_seed_means": [ + 0.0019001337472582236, + 0.004229134528626067, + 0.002199649610168611 + ] + }, + "100": { + "mean_across_seeds": 0.0019877900462112546, + "std_across_seeds": 0.0011851872226942624, + "per_seed_means": [ + 0.001068641186526899, + 0.0036611919594967427, + 0.0012335369926101218 + ] + }, + "200": { + "mean_across_seeds": 0.0012966119715646428, + "std_across_seeds": 0.0012246727872537106, + "per_seed_means": [ + 0.0004260446069141229, + 0.0030285527024049466, + 0.0004352386053748584 + ] + }, + "500": { + "mean_across_seeds": 0.0011126965287621717, + "std_across_seeds": 0.0012704402267073925, + "per_seed_means": [ + 0.0002292340784939976, + 0.002909288679851064, + 0.0001995668279414531 + ] + }, + "1000": { + "mean_across_seeds": 0.0010599333845877684, + "std_across_seeds": 0.0012876423837396504, + "per_seed_means": [ + 0.000164134413607826, + 0.0028808559999985543, + 0.00013480974015692482 + ] + } + }, + "runtime_so_far": 73.1 +} \ No newline at end of file diff --git a/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_mongo.partial.json b/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_mongo.partial.json new file mode 100644 index 0000000000000000000000000000000000000000..1a5e21db94b6eeb4e3240bff4333f7759b8d4fad --- /dev/null +++ b/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_mongo.partial.json @@ -0,0 +1,90 @@ +{ + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "theta": 1000000, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.008414634897166656, + "std_across_seeds": 0.001494959703850088, + "per_seed_means": [ + 0.0070227079143902905, + 0.010488735257725541, + 0.007732461519384136 + ] + }, + "20": { + "mean_across_seeds": 0.005654149828866746, + "std_across_seeds": 0.001134504996687596, + "per_seed_means": [ + 0.004655646131141111, + 0.007241011513785148, + 0.005065791841673975 + ] + }, + "30": { + "mean_across_seeds": 0.0038466191224546895, + "std_across_seeds": 0.0010558909426180547, + "per_seed_means": [ + 0.0030183024186408148, + 0.005336777986764597, + 0.0031847769619586567 + ] + }, + "50": { + "mean_across_seeds": 0.002776305962017634, + "std_across_seeds": 0.0010345564495630884, + "per_seed_means": [ + 0.0019001337472582236, + 0.004229134528626067, + 0.002199649610168611 + ] + }, + "100": { + "mean_across_seeds": 0.0019877900462112546, + "std_across_seeds": 0.0011851872226942624, + "per_seed_means": [ + 0.001068641186526899, + 0.0036611919594967427, + 0.0012335369926101218 + ] + }, + "200": { + "mean_across_seeds": 0.0012966119715646428, + "std_across_seeds": 0.0012246727872537106, + "per_seed_means": [ + 0.0004260446069141229, + 0.0030285527024049466, + 0.0004352386053748584 + ] + }, + "500": { + "mean_across_seeds": 0.0011126965287621717, + "std_across_seeds": 0.0012704402267073925, + "per_seed_means": [ + 0.0002292340784939976, + 0.002909288679851064, + 0.0001995668279414531 + ] + }, + "1000": { + "mean_across_seeds": 0.0010599333845877684, + "std_across_seeds": 0.0012876423837396504, + "per_seed_means": [ + 0.000164134413607826, + 0.0028808559999985543, + 0.00013480974015692482 + ] + } + }, + "runtime_so_far": 73.1 +} \ No newline at end of file diff --git a/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_random.json b/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_random.json new file mode 100644 index 0000000000000000000000000000000000000000..652c285b6798da61c74b11bad108fb6ce38fd144 --- /dev/null +++ b/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_random.json @@ -0,0 +1,90 @@ +{ + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "theta": 1000000, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.0091970660061472, + "std_across_seeds": 0.0008349270605918532, + "per_seed_means": [ + 0.00935791149114569, + 0.010129683905591567, + 0.00810360262170434 + ] + }, + "20": { + "mean_across_seeds": 0.006026908747024006, + "std_across_seeds": 0.0012080059206040254, + "per_seed_means": [ + 0.005178192782526215, + 0.005167258020179967, + 0.007735275438365837 + ] + }, + "30": { + "mean_across_seeds": 0.004927576610094144, + "std_across_seeds": 0.0010036712687790468, + "per_seed_means": [ + 0.0050311802793294195, + 0.003649812405152867, + 0.006101737145800143 + ] + }, + "50": { + "mean_across_seeds": 0.0035441121060608162, + "std_across_seeds": 0.0011052590479342598, + "per_seed_means": [ + 0.0028005982710358995, + 0.0027251605254908404, + 0.005106577521655708 + ] + }, + "100": { + "mean_across_seeds": 0.0018333963030535312, + "std_across_seeds": 0.0005019772099850094, + "per_seed_means": [ + 0.001483295342962568, + 0.0025432772864587604, + 0.0014736162797392657 + ] + }, + "200": { + "mean_across_seeds": 0.0018679075162961252, + "std_across_seeds": 0.0009643543386664822, + "per_seed_means": [ + 0.0007317609019810334, + 0.003089316180557944, + 0.0017826454663493981 + ] + }, + "500": { + "mean_across_seeds": 0.0007167157731924412, + "std_across_seeds": 0.0004266915072455704, + "per_seed_means": [ + 0.00042605371282358344, + 0.0004040783126159416, + 0.0013200152941377989 + ] + }, + "1000": { + "mean_across_seeds": 0.0012025086260756426, + "std_across_seeds": 0.000731409313446818, + "per_seed_means": [ + 0.0012467152821288134, + 0.002075376640811252, + 0.00028543395528686234 + ] + } + }, + "runtime_so_far": 73.8 +} \ No newline at end of file diff --git a/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_random.partial.json b/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_random.partial.json new file mode 100644 index 0000000000000000000000000000000000000000..652c285b6798da61c74b11bad108fb6ce38fd144 --- /dev/null +++ b/data/e4_gamma/mistralai--Mistral-Nemo-Instruct-2407_random.partial.json @@ -0,0 +1,90 @@ +{ + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "theta": 1000000, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.0091970660061472, + "std_across_seeds": 0.0008349270605918532, + "per_seed_means": [ + 0.00935791149114569, + 0.010129683905591567, + 0.00810360262170434 + ] + }, + "20": { + "mean_across_seeds": 0.006026908747024006, + "std_across_seeds": 0.0012080059206040254, + "per_seed_means": [ + 0.005178192782526215, + 0.005167258020179967, + 0.007735275438365837 + ] + }, + "30": { + "mean_across_seeds": 0.004927576610094144, + "std_across_seeds": 0.0010036712687790468, + "per_seed_means": [ + 0.0050311802793294195, + 0.003649812405152867, + 0.006101737145800143 + ] + }, + "50": { + "mean_across_seeds": 0.0035441121060608162, + "std_across_seeds": 0.0011052590479342598, + "per_seed_means": [ + 0.0028005982710358995, + 0.0027251605254908404, + 0.005106577521655708 + ] + }, + "100": { + "mean_across_seeds": 0.0018333963030535312, + "std_across_seeds": 0.0005019772099850094, + "per_seed_means": [ + 0.001483295342962568, + 0.0025432772864587604, + 0.0014736162797392657 + ] + }, + "200": { + "mean_across_seeds": 0.0018679075162961252, + "std_across_seeds": 0.0009643543386664822, + "per_seed_means": [ + 0.0007317609019810334, + 0.003089316180557944, + 0.0017826454663493981 + ] + }, + "500": { + "mean_across_seeds": 0.0007167157731924412, + "std_across_seeds": 0.0004266915072455704, + "per_seed_means": [ + 0.00042605371282358344, + 0.0004040783126159416, + 0.0013200152941377989 + ] + }, + "1000": { + "mean_across_seeds": 0.0012025086260756426, + "std_across_seeds": 0.000731409313446818, + "per_seed_means": [ + 0.0012467152821288134, + 0.002075376640811252, + 0.00028543395528686234 + ] + } + }, + "runtime_so_far": 73.8 +} \ No newline at end of file diff --git a/data/e4_gamma/pade_validation.json b/data/e4_gamma/pade_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..4bb85d5c7d12b4c8f7aaacd829525575e8b70c24 --- /dev/null +++ b/data/e4_gamma/pade_validation.json @@ -0,0 +1,79 @@ +{ + "formula_comparison": [ + { + "name": "pythia-70m", + "theta": 10000, + "T_eval": 2000.0, + "z": 0.282842712474619, + "gamma_obs": 0.7476017873166874, + "r2": 0.9893049417040555, + "gamma_lin": 0.717157287525381, + "err_lin_pct": 4.245163553501351, + "gamma_pade": 0.7522013138014093, + "err_pade_pct": -0.611475465454489, + "gamma_exp": 0.7536383164437648, + "err_exp_pct": -0.8009849015589234, + "notes": "" + }, + { + "name": "Meta-Llama-3-8B", + "theta": 500000, + "T_eval": 2000.0, + "z": 0.00565685424949238, + "gamma_obs": 1.0454762537473639, + "r2": 0.996718622313285, + "gamma_lin": 0.9943431457505076, + "err_lin_pct": 5.1424006104313325, + "gamma_pade": 0.9943591006233126, + "err_pade_pct": 5.140713560323286, + "gamma_exp": 0.9943591156232368, + "err_exp_pct": 5.140711974273832, + "notes": "" + }, + { + "name": "Qwen2.5-7B", + "theta": 1000000, + "T_eval": 2000.0, + "z": 0.00282842712474619, + "gamma_obs": 0.9966953735480816, + "r2": 0.9963935840252253, + "gamma_lin": 0.9971715728752538, + "err_lin_pct": -0.047755004266634545, + "gamma_pade": 0.9971755672263881, + "err_pade_pct": -0.04815537946263395, + "gamma_exp": 0.9971755691066828, + "err_exp_pct": -0.048155567933881434, + "notes": "" + }, + { + "name": "Llama-2-7b-hf", + "theta": 10000, + "T_eval": 2000.0, + "z": 0.282842712474619, + "gamma_obs": 0.2870574377368437, + "r2": 0.881768027724978, + "gamma_lin": 0.717157287525381, + "err_lin_pct": -59.97287586278841, + "gamma_pade": 0.7522013138014093, + "err_pade_pct": -61.83768461050169, + "gamma_exp": 0.7536383164437648, + "err_exp_pct": -61.91045074626808, + "notes": "artifact" + }, + { + "name": "gemma-2-9b-it", + "theta": 10000, + "T_eval": 2000.0, + "z": 0.282842712474619, + "gamma_obs": 0.6276459084140061, + "r2": 0.9846855297721232, + "gamma_lin": 0.717157287525381, + "err_lin_pct": -12.48141525832392, + "gamma_pade": 0.7522013138014093, + "err_pade_pct": -16.55878593962246, + "gamma_exp": 0.7536383164437648, + "err_exp_pct": -16.717887782601878, + "notes": "partial" + } + ] +} \ No newline at end of file diff --git a/data/e4_gamma/tiiuae--falcon-7b_mongo.json b/data/e4_gamma/tiiuae--falcon-7b_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..5576cf8b9dfd72e2bd5ede82502404dcbe331eff --- /dev/null +++ b/data/e4_gamma/tiiuae--falcon-7b_mongo.json @@ -0,0 +1,90 @@ +{ + "model": "tiiuae/falcon-7b", + "theta": 10000, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.007024754305602982, + "std_across_seeds": 0.000281022954489519, + "per_seed_means": [ + 0.006925813024087499, + 0.006740880052093417, + 0.007407569840628033 + ] + }, + "20": { + "mean_across_seeds": 0.004292038934331181, + "std_across_seeds": 0.00011120533415042138, + "per_seed_means": [ + 0.0043627519090659915, + 0.004135028446132007, + 0.004378336447795543 + ] + }, + "30": { + "mean_across_seeds": 0.00297348856072252, + "std_across_seeds": 0.00027029469153299083, + "per_seed_means": [ + 0.0031900272436905653, + 0.0031380233238451185, + 0.0025924151146318763 + ] + }, + "50": { + "mean_across_seeds": 0.001992317959116513, + "std_across_seeds": 0.00015055028320812606, + "per_seed_means": [ + 0.002133448749470214, + 0.002059809279356462, + 0.001783695848522863 + ] + }, + "100": { + "mean_across_seeds": 0.0011603233351423924, + "std_across_seeds": 0.00016550930363410277, + "per_seed_means": [ + 0.0012420231260087651, + 0.0013094309415707054, + 0.0009295159378477062 + ] + }, + "200": { + "mean_across_seeds": 0.0005808056467988839, + "std_across_seeds": 0.0001326133160516149, + "per_seed_means": [ + 0.0006070260007982142, + 0.000728517754226535, + 0.0004068731853719025 + ] + }, + "500": { + "mean_across_seeds": 0.00018795115841688432, + "std_across_seeds": 5.663061902735787e-05, + "per_seed_means": [ + 0.00019070447501993233, + 0.00025589156116742135, + 0.00011725743906329929 + ] + }, + "1000": { + "mean_across_seeds": 0.00013777538599242688, + "std_across_seeds": 3.4127848210976066e-05, + "per_seed_means": [ + 0.0001343877820909256, + 0.00018116400944563792, + 9.777436644071713e-05 + ] + } + }, + "runtime_so_far": 38.1 +} \ No newline at end of file diff --git a/data/e4_gamma/tiiuae--falcon-7b_mongo.partial.json b/data/e4_gamma/tiiuae--falcon-7b_mongo.partial.json new file mode 100644 index 0000000000000000000000000000000000000000..5576cf8b9dfd72e2bd5ede82502404dcbe331eff --- /dev/null +++ b/data/e4_gamma/tiiuae--falcon-7b_mongo.partial.json @@ -0,0 +1,90 @@ +{ + "model": "tiiuae/falcon-7b", + "theta": 10000, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.007024754305602982, + "std_across_seeds": 0.000281022954489519, + "per_seed_means": [ + 0.006925813024087499, + 0.006740880052093417, + 0.007407569840628033 + ] + }, + "20": { + "mean_across_seeds": 0.004292038934331181, + "std_across_seeds": 0.00011120533415042138, + "per_seed_means": [ + 0.0043627519090659915, + 0.004135028446132007, + 0.004378336447795543 + ] + }, + "30": { + "mean_across_seeds": 0.00297348856072252, + "std_across_seeds": 0.00027029469153299083, + "per_seed_means": [ + 0.0031900272436905653, + 0.0031380233238451185, + 0.0025924151146318763 + ] + }, + "50": { + "mean_across_seeds": 0.001992317959116513, + "std_across_seeds": 0.00015055028320812606, + "per_seed_means": [ + 0.002133448749470214, + 0.002059809279356462, + 0.001783695848522863 + ] + }, + "100": { + "mean_across_seeds": 0.0011603233351423924, + "std_across_seeds": 0.00016550930363410277, + "per_seed_means": [ + 0.0012420231260087651, + 0.0013094309415707054, + 0.0009295159378477062 + ] + }, + "200": { + "mean_across_seeds": 0.0005808056467988839, + "std_across_seeds": 0.0001326133160516149, + "per_seed_means": [ + 0.0006070260007982142, + 0.000728517754226535, + 0.0004068731853719025 + ] + }, + "500": { + "mean_across_seeds": 0.00018795115841688432, + "std_across_seeds": 5.663061902735787e-05, + "per_seed_means": [ + 0.00019070447501993233, + 0.00025589156116742135, + 0.00011725743906329929 + ] + }, + "1000": { + "mean_across_seeds": 0.00013777538599242688, + "std_across_seeds": 3.4127848210976066e-05, + "per_seed_means": [ + 0.0001343877820909256, + 0.00018116400944563792, + 9.777436644071713e-05 + ] + } + }, + "runtime_so_far": 38.1 +} \ No newline at end of file diff --git a/data/e4_gamma/tiiuae--falcon-7b_random.json b/data/e4_gamma/tiiuae--falcon-7b_random.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1f67610708cd4b27768d948658d823329045fc --- /dev/null +++ b/data/e4_gamma/tiiuae--falcon-7b_random.json @@ -0,0 +1,90 @@ +{ + "model": "tiiuae/falcon-7b", + "theta": 10000, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.006471662033970157, + "std_across_seeds": 0.00019798075527525136, + "per_seed_means": [ + 0.006681017264102896, + 0.0062059801956638695, + 0.0065279886421437065 + ] + }, + "20": { + "mean_across_seeds": 0.004763770298514929, + "std_across_seeds": 3.914722336952796e-05, + "per_seed_means": [ + 0.004717290437159439, + 0.004760962685880562, + 0.004813057772504787 + ] + }, + "30": { + "mean_across_seeds": 0.003051769348482291, + "std_across_seeds": 0.00022639859457039847, + "per_seed_means": [ + 0.00299823039288943, + 0.0028051623996968073, + 0.0033519152528606357 + ] + }, + "50": { + "mean_across_seeds": 0.0023731324412963454, + "std_across_seeds": 1.6489268443542254e-05, + "per_seed_means": [ + 0.0023677307126733164, + 0.002356187442007164, + 0.0023954791692085563 + ] + }, + "100": { + "mean_across_seeds": 0.0015426954483458153, + "std_across_seeds": 3.925228359984241e-05, + "per_seed_means": [ + 0.0015002908720634879, + 0.001532873526836435, + 0.0015949219461375227 + ] + }, + "200": { + "mean_across_seeds": 0.0006482043851348053, + "std_across_seeds": 2.449942868397519e-05, + "per_seed_means": [ + 0.0006654684979002923, + 0.0006135570290886486, + 0.0006655876284154753 + ] + }, + "500": { + "mean_across_seeds": 0.00019178291543438613, + "std_across_seeds": 3.4106950473790724e-06, + "per_seed_means": [ + 0.00019566926906312195, + 0.00019231389558020358, + 0.00018736558165983298 + ] + }, + "1000": { + "mean_across_seeds": 0.0001726856403547572, + "std_across_seeds": 4.839612172465104e-06, + "per_seed_means": [ + 0.00016680734326655512, + 0.0001725888398747581, + 0.00017866073792295842 + ] + } + }, + "runtime_so_far": 3342.7 +} \ No newline at end of file diff --git a/data/e4_gamma/tiiuae--falcon-7b_random.partial.json b/data/e4_gamma/tiiuae--falcon-7b_random.partial.json new file mode 100644 index 0000000000000000000000000000000000000000..8d1f67610708cd4b27768d948658d823329045fc --- /dev/null +++ b/data/e4_gamma/tiiuae--falcon-7b_random.partial.json @@ -0,0 +1,90 @@ +{ + "model": "tiiuae/falcon-7b", + "theta": 10000, + "status": "partial", + "distances_done": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "attn_by_distance": { + "10": { + "mean_across_seeds": 0.006471662033970157, + "std_across_seeds": 0.00019798075527525136, + "per_seed_means": [ + 0.006681017264102896, + 0.0062059801956638695, + 0.0065279886421437065 + ] + }, + "20": { + "mean_across_seeds": 0.004763770298514929, + "std_across_seeds": 3.914722336952796e-05, + "per_seed_means": [ + 0.004717290437159439, + 0.004760962685880562, + 0.004813057772504787 + ] + }, + "30": { + "mean_across_seeds": 0.003051769348482291, + "std_across_seeds": 0.00022639859457039847, + "per_seed_means": [ + 0.00299823039288943, + 0.0028051623996968073, + 0.0033519152528606357 + ] + }, + "50": { + "mean_across_seeds": 0.0023731324412963454, + "std_across_seeds": 1.6489268443542254e-05, + "per_seed_means": [ + 0.0023677307126733164, + 0.002356187442007164, + 0.0023954791692085563 + ] + }, + "100": { + "mean_across_seeds": 0.0015426954483458153, + "std_across_seeds": 3.925228359984241e-05, + "per_seed_means": [ + 0.0015002908720634879, + 0.001532873526836435, + 0.0015949219461375227 + ] + }, + "200": { + "mean_across_seeds": 0.0006482043851348053, + "std_across_seeds": 2.449942868397519e-05, + "per_seed_means": [ + 0.0006654684979002923, + 0.0006135570290886486, + 0.0006655876284154753 + ] + }, + "500": { + "mean_across_seeds": 0.00019178291543438613, + "std_across_seeds": 3.4106950473790724e-06, + "per_seed_means": [ + 0.00019566926906312195, + 0.00019231389558020358, + 0.00018736558165983298 + ] + }, + "1000": { + "mean_across_seeds": 0.0001726856403547572, + "std_across_seeds": 4.839612172465104e-06, + "per_seed_means": [ + 0.00016680734326655512, + 0.0001725888398747581, + 0.00017866073792295842 + ] + } + }, + "runtime_so_far": 3342.7 +} \ No newline at end of file diff --git a/data/e5_amplitude/EleutherAI--pythia-160m.json b/data/e5_amplitude/EleutherAI--pythia-160m.json new file mode 100644 index 0000000000000000000000000000000000000000..7f3f5c2a01f32e08a09035194eed7ffd1fa43394 --- /dev/null +++ b/data/e5_amplitude/EleutherAI--pythia-160m.json @@ -0,0 +1,67 @@ +{ + "model": "EleutherAI/pythia-160m", + "theta": 10000, + "C_theory": 9.2103, + "C_measured": NaN, + "C_error_pct": null, + "A_bar_measured": NaN, + "A_bar_predicted": 2.3026, + "A_std": NaN, + "gamma_from_weights": NaN, + "gamma_from_E4": null, + "d_head": 64, + "n_heads": 12, + "n_layers_captured": 12, + "n_samples": 50, + "n_tokens": 128, + "per_layer_A_bar": [ + 1.6555, + 2.0901, + 2.2677, + 3.2325, + 9.0963, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + "per_dim_A_bar": [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + "runtime_seconds": 4.9, + "decision": "FAILED: no amplitudes captured" +} \ No newline at end of file diff --git a/data/e5_amplitude/EleutherAI--pythia-1b.json b/data/e5_amplitude/EleutherAI--pythia-1b.json new file mode 100644 index 0000000000000000000000000000000000000000..9645d595f50546ce44820ec7b6a9f8f399b8f832 --- /dev/null +++ b/data/e5_amplitude/EleutherAI--pythia-1b.json @@ -0,0 +1,167 @@ +{ + "model": "EleutherAI/pythia-1b", + "theta": 10000, + "C_theory": 9.2103, + "C_measured": 5.3827, + "C_error_pct": 41.56, + "A_bar_measured": 0.6728, + "A_bar_predicted": 1.1513, + "A_std": 0.1333, + "gamma_from_weights": 0.5844, + "gamma_from_E4": null, + "d_head": 256, + "n_heads": 8, + "n_layers_captured": 16, + "n_samples": 50, + "n_tokens": 128, + "per_layer_A_bar": [ + 0.3544, + 0.9359, + 0.5252, + 0.6308, + 0.7435, + 0.7507, + 0.7498, + 0.8639, + 0.6975, + 0.7034, + 0.7169, + 0.6854, + 0.7002, + 0.567, + 0.5582, + 0.5827 + ], + "per_dim_A_bar": [ + 0.5197, + 0.5297, + 0.7485, + 0.6562, + 1.2286, + 1.1744, + 1.4516, + 1.1216, + 1.403, + 1.3567, + 0.8314, + 4.216, + 2.3154, + 0.6761, + 0.5118, + 0.4868, + 0.5677, + 0.5554, + 0.7208, + 0.7326, + 1.0284, + 1.2848, + 1.3934, + 0.9543, + 1.3305, + 1.4842, + 0.8558, + 3.9297, + 1.7067, + 0.6312, + 0.5256, + 0.5136, + 0.4719, + 0.4933, + 0.5203, + 0.5287, + 0.4902, + 0.4995, + 0.564, + 0.5013, + 0.4983, + 0.5108, + 0.4846, + 0.5323, + 0.516, + 0.569, + 0.5089, + 0.5157, + 0.5, + 0.499, + 0.5261, + 0.4892, + 0.5036, + 0.5102, + 0.5317, + 0.5105, + 0.4863, + 0.506, + 0.5216, + 0.5126, + 0.5572, + 0.4909, + 0.5639, + 0.5304, + 0.4891, + 0.4871, + 0.505, + 0.5071, + 0.5229, + 0.5203, + 0.5044, + 0.4743, + 0.4929, + 0.4886, + 0.4865, + 0.4777, + 0.5538, + 0.4605, + 0.5321, + 0.5036, + 0.483, + 0.5434, + 0.49, + 0.5189, + 0.5009, + 0.5659, + 0.5293, + 0.5283, + 0.5435, + 0.5007, + 0.4876, + 0.4711, + 0.4983, + 0.5175, + 0.5002, + 0.4917, + 0.5304, + 0.5179, + 0.4744, + 0.499, + 0.4754, + 0.5034, + 0.491, + 0.5003, + 0.5077, + 0.4741, + 0.4789, + 0.5103, + 0.5325, + 0.4993, + 0.5317, + 0.5363, + 0.4987, + 0.4933, + 0.5261, + 0.4833, + 0.5038, + 0.4729, + 0.5179, + 0.4909, + 0.5032, + 0.4968, + 0.4711, + 0.5339, + 0.5008, + 0.4987, + 0.5051, + 0.5022 + ], + "runtime_seconds": 21.6, + "decision": "C_measured=5.383 vs C_theory=9.210: ANOMALY (err=41.6%)" +} \ No newline at end of file diff --git a/data/e5_amplitude/EleutherAI--pythia-2.8b.json b/data/e5_amplitude/EleutherAI--pythia-2.8b.json new file mode 100644 index 0000000000000000000000000000000000000000..380041a947a58f571e809f2c745a25f8e4b86c04 --- /dev/null +++ b/data/e5_amplitude/EleutherAI--pythia-2.8b.json @@ -0,0 +1,95 @@ +{ + "model": "EleutherAI/pythia-2.8b", + "theta": 10000, + "C_theory": 9.2103, + "C_measured": NaN, + "C_error_pct": null, + "A_bar_measured": NaN, + "A_bar_predicted": 2.0595, + "A_std": NaN, + "gamma_from_weights": NaN, + "gamma_from_E4": null, + "d_head": 80, + "n_heads": 32, + "n_layers_captured": 32, + "n_samples": 50, + "n_tokens": 128, + "per_layer_A_bar": [ + 0.4882, + 0.9339, + 0.5753, + 0.7858, + 0.7913, + 0.9489, + 0.9047, + 0.8892, + 1.1647, + 1.0917, + 1.3604, + 1.2763, + 1.322, + 1.6303, + 1.844, + 2.0808, + 2.0532, + 2.7395, + 3.2109, + 4.2194, + 7.3082, + 8.0346, + 17.3404, + 30.0029, + 38.4212, + 61.0449, + 73.5542, + 51.2656, + 68.1474, + NaN, + NaN, + NaN + ], + "per_dim_A_bar": [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + "runtime_seconds": 33.1, + "decision": "FAILED: no amplitudes captured" +} \ No newline at end of file diff --git a/data/e5_amplitude/EleutherAI--pythia-410m.json b/data/e5_amplitude/EleutherAI--pythia-410m.json new file mode 100644 index 0000000000000000000000000000000000000000..21968193533f3e08c4e5629cacd35df37346685b --- /dev/null +++ b/data/e5_amplitude/EleutherAI--pythia-410m.json @@ -0,0 +1,79 @@ +{ + "model": "EleutherAI/pythia-410m", + "theta": 10000, + "C_theory": 9.2103, + "C_measured": NaN, + "C_error_pct": null, + "A_bar_measured": NaN, + "A_bar_predicted": 2.3026, + "A_std": NaN, + "gamma_from_weights": NaN, + "gamma_from_E4": null, + "d_head": 64, + "n_heads": 16, + "n_layers_captured": 24, + "n_samples": 50, + "n_tokens": 128, + "per_layer_A_bar": [ + 0.7246, + 1.0436, + 0.717, + 0.8015, + 0.6713, + 1.2467, + 1.1805, + 1.6927, + 1.4511, + 1.4148, + 2.546, + 2.1705, + 2.3569, + 2.9459, + 7.8879, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + "per_dim_A_bar": [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + "runtime_seconds": 10.5, + "decision": "FAILED: no amplitudes captured" +} \ No newline at end of file diff --git a/data/e5_amplitude/EleutherAI--pythia-70m.json b/data/e5_amplitude/EleutherAI--pythia-70m.json new file mode 100644 index 0000000000000000000000000000000000000000..8d6ba02885257ab0fc6e0e568cc70934a577a6b2 --- /dev/null +++ b/data/e5_amplitude/EleutherAI--pythia-70m.json @@ -0,0 +1,61 @@ +{ + "model": "EleutherAI/pythia-70m", + "theta": 10000, + "C_theory": 9.2103, + "C_measured": NaN, + "C_error_pct": null, + "A_bar_measured": NaN, + "A_bar_predicted": 2.3026, + "A_std": NaN, + "gamma_from_weights": NaN, + "gamma_from_E4": null, + "d_head": 64, + "n_heads": 8, + "n_layers_captured": 6, + "n_samples": 50, + "n_tokens": 128, + "per_layer_A_bar": [ + 2.0719, + 2.1383, + 2.5327, + 65.7903, + NaN, + NaN + ], + "per_dim_A_bar": [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + "runtime_seconds": 2.5, + "decision": "FAILED: no amplitudes captured" +} \ No newline at end of file diff --git a/data/e5_amplitude/gpt2-xl.json b/data/e5_amplitude/gpt2-xl.json new file mode 100644 index 0000000000000000000000000000000000000000..eb8d71d1ab8e83f0b6b2b558be158921dd92f05e --- /dev/null +++ b/data/e5_amplitude/gpt2-xl.json @@ -0,0 +1,103 @@ +{ + "model": "gpt2-xl", + "theta": null, + "C_theory": 9.2103, + "C_measured": 5.1924, + "C_error_pct": 43.62, + "A_bar_measured": 1.2981, + "A_bar_predicted": 2.3026, + "A_std": 0.3159, + "gamma_from_weights": null, + "gamma_from_E4": null, + "d_head": 64, + "n_heads": 25, + "n_layers_captured": 48, + "n_samples": 50, + "n_tokens": 128, + "per_layer_A_bar": [ + 0.1826, + 0.5438, + 0.7825, + 1.1647, + 1.3698, + 1.5361, + 1.5838, + 1.5415, + 1.5964, + 1.4611, + 1.3209, + 1.3818, + 1.6553, + 1.6439, + 1.6256, + 1.4887, + 1.5875, + 1.6609, + 1.5901, + 1.3969, + 1.4378, + 1.5796, + 1.6294, + 1.4923, + 1.4155, + 1.4922, + 1.5155, + 1.403, + 1.3828, + 1.2529, + 1.3058, + 1.3096, + 1.3026, + 1.2972, + 1.3793, + 1.277, + 1.299, + 1.2757, + 1.1785, + 1.2169, + 1.1775, + 1.1916, + 1.111, + 1.109, + 1.066, + 0.8684, + 0.6571, + 0.572 + ], + "per_dim_A_bar": [ + 1.2784, + 1.3119, + 1.2721, + 1.2502, + 1.3204, + 1.3095, + 1.2482, + 1.26, + 1.2387, + 1.3523, + 1.3001, + 1.2819, + 1.303, + 1.2809, + 1.3204, + 1.3093, + 1.34, + 1.3438, + 1.2853, + 1.286, + 1.2813, + 1.3216, + 1.3054, + 1.2953, + 1.3246, + 1.3335, + 1.3003, + 1.3287, + 1.2619, + 1.3065, + 1.263, + 1.3251 + ], + "runtime_seconds": 34.2, + "decision": "C_measured=5.192 vs C_theory=9.210: ANOMALY (err=43.6%)" +} \ No newline at end of file diff --git a/data/e5_amplitude/gpt2.json b/data/e5_amplitude/gpt2.json new file mode 100644 index 0000000000000000000000000000000000000000..399b58f08d4d93e20c6cb9afb0c89788a613fdd9 --- /dev/null +++ b/data/e5_amplitude/gpt2.json @@ -0,0 +1,67 @@ +{ + "model": "gpt2", + "theta": null, + "C_theory": 9.2103, + "C_measured": 8.2494, + "C_error_pct": 10.43, + "A_bar_measured": 2.0624, + "A_bar_predicted": 2.3026, + "A_std": 1.1068, + "gamma_from_weights": null, + "gamma_from_E4": null, + "d_head": 64, + "n_heads": 12, + "n_layers_captured": 12, + "n_samples": 50, + "n_tokens": 128, + "per_layer_A_bar": [ + 1.1999, + 0.8809, + 2.5485, + 4.4418, + 4.155, + 2.0613, + 1.955, + 1.9619, + 1.7539, + 1.6041, + 1.2899, + 0.896 + ], + "per_dim_A_bar": [ + 1.9641, + 1.9675, + 1.7644, + 2.4292, + 2.0933, + 1.8641, + 1.9431, + 2.0676, + 2.1309, + 1.7681, + 2.2024, + 2.2397, + 1.9954, + 2.1919, + 2.3473, + 1.9186, + 1.7409, + 2.103, + 2.3969, + 1.8852, + 2.1082, + 2.0003, + 2.4433, + 2.1634, + 1.8313, + 2.0641, + 1.8851, + 2.0088, + 2.4169, + 1.901, + 2.1619, + 1.9975 + ], + "runtime_seconds": 4.9, + "decision": "C_measured=8.249 vs C_theory=9.210: ANOMALY (err=10.4%)" +} \ No newline at end of file diff --git a/data/e5_cardy/E5_EleutherAI--pythia-1.4b_mongo.csv b/data/e5_cardy/E5_EleutherAI--pythia-1.4b_mongo.csv new file mode 100644 index 0000000000000000000000000000000000000000..48aa0625a3383b1270cb4873608d038e1bb100a1 --- /dev/null +++ b/data/e5_cardy/E5_EleutherAI--pythia-1.4b_mongo.csv @@ -0,0 +1,25 @@ +layer,mean_H,delta_H +0,4.193755,-4.705575 +1,3.450005,-5.449325 +2,4.346648,-4.552683 +3,3.740251,-5.159080 +4,2.680097,-6.219234 +5,2.158266,-6.741065 +6,2.143276,-6.756054 +7,2.209393,-6.689938 +8,2.296192,-6.603138 +9,2.613837,-6.285494 +10,2.186772,-6.712559 +11,2.432494,-6.466836 +12,2.057389,-6.841941 +13,1.844082,-7.055249 +14,1.694679,-7.204652 +15,1.533046,-7.366285 +16,0.000000,-8.899331 +17,0.000000,-8.899331 +18,0.000000,-8.899331 +19,0.000000,-8.899331 +20,0.000000,-8.899331 +21,0.000000,-8.899331 +22,0.000000,-8.899331 +23,0.000000,-8.899331 diff --git a/data/e5_cardy/E5_EleutherAI--pythia-1.4b_mongo.json b/data/e5_cardy/E5_EleutherAI--pythia-1.4b_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..6b5fd74f9111bcbe23501b3cbe0249e214a5135e --- /dev/null +++ b/data/e5_cardy/E5_EleutherAI--pythia-1.4b_mongo.json @@ -0,0 +1,158 @@ +{ + "model": "EleutherAI/pythia-1.4b", + "corpus": "mongo", + "theta": 10000, + "T_eval": 512, + "theta_eff_pade": 10362.038671967512, + "H_cardy": 8.899330689182712, + "H_obs": 1.7325076003383073, + "delta_H": -7.166823088844405, + "holographic_efficiency": 0.0007717706876052656, + "n_prompts": 30, + "per_layer": [ + { + "layer": 0, + "mean_H": 4.193755467049778, + "std_H": 1.0856597340128886, + "n": 480 + }, + { + "layer": 1, + "mean_H": 3.4500054485785463, + "std_H": 1.3905631580207733, + "n": 480 + }, + { + "layer": 2, + "mean_H": 4.346647810315092, + "std_H": 1.0159820844941474, + "n": 480 + }, + { + "layer": 3, + "mean_H": 3.7402507920128603, + "std_H": 1.2287588389329953, + "n": 480 + }, + { + "layer": 4, + "mean_H": 2.680097132641822, + "std_H": 1.2170667110218805, + "n": 480 + }, + { + "layer": 5, + "mean_H": 2.1582659481947, + "std_H": 1.0975088905923274, + "n": 480 + }, + { + "layer": 6, + "mean_H": 2.1432762716586393, + "std_H": 1.0619770878335983, + "n": 480 + }, + { + "layer": 7, + "mean_H": 2.2093931287527084, + "std_H": 1.004682714406244, + "n": 480 + }, + { + "layer": 8, + "mean_H": 2.296192371658981, + "std_H": 1.0124081724838316, + "n": 480 + }, + { + "layer": 9, + "mean_H": 2.6138366468871634, + "std_H": 1.052677434375479, + "n": 480 + }, + { + "layer": 10, + "mean_H": 2.186771517836799, + "std_H": 1.2075750426076084, + "n": 480 + }, + { + "layer": 11, + "mean_H": 2.4324943229090423, + "std_H": 1.1918171321322655, + "n": 480 + }, + { + "layer": 12, + "mean_H": 2.0573892562882974, + "std_H": 1.1394072807684963, + "n": 480 + }, + { + "layer": 13, + "mean_H": 1.8440815550036267, + "std_H": 1.1311256699162422, + "n": 480 + }, + { + "layer": 14, + "mean_H": 1.694679054266271, + "std_H": 1.1587010930864075, + "n": 480 + }, + { + "layer": 15, + "mean_H": 1.5330456840650488, + "std_H": 1.1291099420898785, + "n": 480 + }, + { + "layer": 16, + "mean_H": 0.0, + "std_H": 0.0, + "n": 480 + }, + { + "layer": 17, + "mean_H": 0.0, + "std_H": 0.0, + "n": 480 + }, + { + "layer": 18, + "mean_H": 0.0, + "std_H": 0.0, + "n": 480 + }, + { + "layer": 19, + "mean_H": 0.0, + "std_H": 0.0, + "n": 480 + }, + { + "layer": 20, + "mean_H": 0.0, + "std_H": 0.0, + "n": 480 + }, + { + "layer": 21, + "mean_H": 0.0, + "std_H": 0.0, + "n": 480 + }, + { + "layer": 22, + "mean_H": 0.0, + "std_H": 0.0, + "n": 480 + }, + { + "layer": 23, + "mean_H": 0.0, + "std_H": 0.0, + "n": 480 + } + ] +} \ No newline at end of file diff --git a/data/e5_cardy/E5_EleutherAI--pythia-1b_mongo.csv b/data/e5_cardy/E5_EleutherAI--pythia-1b_mongo.csv new file mode 100644 index 0000000000000000000000000000000000000000..3fba31102e22632bacec6969eebcc4fce95d0a87 --- /dev/null +++ b/data/e5_cardy/E5_EleutherAI--pythia-1b_mongo.csv @@ -0,0 +1,17 @@ +layer,mean_H,delta_H +0,4.439314,-4.460017 +1,3.692324,-5.207007 +2,3.554057,-5.345274 +3,2.974438,-5.924893 +4,2.381632,-6.517699 +5,2.636901,-6.262430 +6,2.456597,-6.442734 +7,1.950147,-6.949184 +8,2.788092,-6.111239 +9,2.961767,-5.937564 +10,2.142983,-6.756348 +11,2.530935,-6.368395 +12,2.245301,-6.654030 +13,2.067567,-6.831763 +14,1.932013,-6.967318 +15,2.135496,-6.763835 diff --git a/data/e5_cardy/E5_EleutherAI--pythia-1b_mongo.json b/data/e5_cardy/E5_EleutherAI--pythia-1b_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..0669969fb757e015cc1c3e8f56618353a8eb6a98 --- /dev/null +++ b/data/e5_cardy/E5_EleutherAI--pythia-1b_mongo.json @@ -0,0 +1,110 @@ +{ + "model": "EleutherAI/pythia-1b", + "corpus": "mongo", + "theta": 10000, + "T_eval": 512, + "theta_eff_pade": 10362.038671967512, + "H_cardy": 8.899330689182712, + "H_obs": 2.680597666463958, + "delta_H": -6.218733022718753, + "holographic_efficiency": 0.001991767143036238, + "n_prompts": 30, + "per_layer": [ + { + "layer": 0, + "mean_H": 4.439313767105341, + "std_H": 0.8326861694360956, + "n": 240 + }, + { + "layer": 1, + "mean_H": 3.6923237988414863, + "std_H": 1.1985485479976765, + "n": 240 + }, + { + "layer": 2, + "mean_H": 3.5540570341050626, + "std_H": 1.084168824765083, + "n": 240 + }, + { + "layer": 3, + "mean_H": 2.974437540236977, + "std_H": 1.6214941855944098, + "n": 240 + }, + { + "layer": 4, + "mean_H": 2.3816321183616918, + "std_H": 1.066572979624109, + "n": 240 + }, + { + "layer": 5, + "mean_H": 2.6369010283301275, + "std_H": 1.1879534389965174, + "n": 240 + }, + { + "layer": 6, + "mean_H": 2.4565971842346093, + "std_H": 1.1333641813489372, + "n": 240 + }, + { + "layer": 7, + "mean_H": 1.9501471442713714, + "std_H": 1.0054569710423744, + "n": 240 + }, + { + "layer": 8, + "mean_H": 2.788091965143879, + "std_H": 1.0012327315672245, + "n": 240 + }, + { + "layer": 9, + "mean_H": 2.961766531566779, + "std_H": 1.0770823452589096, + "n": 240 + }, + { + "layer": 10, + "mean_H": 2.142982858543595, + "std_H": 0.8937415789386295, + "n": 240 + }, + { + "layer": 11, + "mean_H": 2.530935339257121, + "std_H": 1.1354011653676355, + "n": 240 + }, + { + "layer": 12, + "mean_H": 2.2453007698369523, + "std_H": 0.9927599448139722, + "n": 240 + }, + { + "layer": 13, + "mean_H": 2.067567472252995, + "std_H": 1.0966607033966111, + "n": 240 + }, + { + "layer": 14, + "mean_H": 1.9320125193024675, + "std_H": 0.9780805762865461, + "n": 240 + }, + { + "layer": 15, + "mean_H": 2.1354955920328695, + "std_H": 1.1861320500665091, + "n": 240 + } + ] +} \ No newline at end of file diff --git a/data/e5_cardy/E5_EleutherAI--pythia-70m_mongo.csv b/data/e5_cardy/E5_EleutherAI--pythia-70m_mongo.csv new file mode 100644 index 0000000000000000000000000000000000000000..df7c74a1634b82969141675fb6ec487c0f863c78 --- /dev/null +++ b/data/e5_cardy/E5_EleutherAI--pythia-70m_mongo.csv @@ -0,0 +1,7 @@ +layer,mean_H,delta_H +0,3.546627,-5.352703 +1,3.374105,-5.525226 +2,2.742319,-6.157012 +3,1.599187,-7.300144 +4,0.000000,-8.899331 +5,0.000000,-8.899331 diff --git a/data/e5_cardy/E5_EleutherAI--pythia-70m_mongo.json b/data/e5_cardy/E5_EleutherAI--pythia-70m_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..97ca3fef93f9a251c832dd23240006c9ec70fca3 --- /dev/null +++ b/data/e5_cardy/E5_EleutherAI--pythia-70m_mongo.json @@ -0,0 +1,50 @@ +{ + "model": "EleutherAI/pythia-70m", + "corpus": "mongo", + "theta": 10000, + "T_eval": 512, + "theta_eff_pade": 10362.038671967512, + "H_cardy": 8.899330689182712, + "H_obs": 1.8770396334714152, + "delta_H": -7.022291055711296, + "holographic_efficiency": 0.0008917800328746847, + "n_prompts": 30, + "per_layer": [ + { + "layer": 0, + "mean_H": 3.5466272787967075, + "std_H": 1.294231335784017, + "n": 240 + }, + { + "layer": 1, + "mean_H": 3.374105114986499, + "std_H": 1.1471069310594448, + "n": 240 + }, + { + "layer": 2, + "mean_H": 2.7423187561333178, + "std_H": 1.3135643487430346, + "n": 240 + }, + { + "layer": 3, + "mean_H": 1.599186650911967, + "std_H": 1.6446712356629896, + "n": 240 + }, + { + "layer": 4, + "mean_H": 0.0, + "std_H": 0.0, + "n": 240 + }, + { + "layer": 5, + "mean_H": 0.0, + "std_H": 0.0, + "n": 240 + } + ] +} \ No newline at end of file diff --git a/data/e5_cardy/E5_meta-llama--Llama-2-7b-hf_mongo.csv b/data/e5_cardy/E5_meta-llama--Llama-2-7b-hf_mongo.csv new file mode 100644 index 0000000000000000000000000000000000000000..116dff2ece6d6909abb0561931fe691c446dc538 --- /dev/null +++ b/data/e5_cardy/E5_meta-llama--Llama-2-7b-hf_mongo.csv @@ -0,0 +1,33 @@ +layer,mean_H,delta_H +0,5.781910,-3.117421 +1,4.889605,-4.009726 +2,1.463313,-7.436018 +3,1.375873,-7.523458 +4,1.590204,-7.309127 +5,1.711600,-7.187731 +6,1.912080,-6.987251 +7,1.806940,-7.092391 +8,2.065460,-6.833871 +9,2.271931,-6.627400 +10,2.514428,-6.384903 +11,2.476174,-6.423157 +12,2.430441,-6.468890 +13,2.260128,-6.639202 +14,2.460239,-6.439092 +15,2.268332,-6.630998 +16,2.205484,-6.693847 +17,1.937975,-6.961356 +18,1.709271,-7.190060 +19,1.755873,-7.143458 +20,1.852089,-7.047242 +21,1.597261,-7.302070 +22,1.585322,-7.314009 +23,1.400611,-7.498719 +24,1.588280,-7.311051 +25,1.349389,-7.549941 +26,1.660862,-7.238469 +27,1.304400,-7.594931 +28,1.416922,-7.482408 +29,1.706087,-7.193243 +30,1.452686,-7.446644 +31,2.202860,-6.696471 diff --git a/data/e5_cardy/E5_meta-llama--Llama-2-7b-hf_mongo.json b/data/e5_cardy/E5_meta-llama--Llama-2-7b-hf_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..6ad6d60a7791fa06c59e676379255c6c9f6b950f --- /dev/null +++ b/data/e5_cardy/E5_meta-llama--Llama-2-7b-hf_mongo.json @@ -0,0 +1,206 @@ +{ + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "mongo", + "theta": 10000, + "T_eval": 512, + "theta_eff_pade": 10362.038671967512, + "H_cardy": 8.899330689182712, + "H_obs": 2.062625857736839, + "delta_H": -6.836704831445873, + "holographic_efficiency": 0.0010736353867623126, + "n_prompts": 20, + "per_layer": [ + { + "layer": 0, + "mean_H": 5.781909537315369, + "std_H": 0.5146723490855796, + "n": 640 + }, + { + "layer": 1, + "mean_H": 4.889604668784886, + "std_H": 0.8965258994021935, + "n": 640 + }, + { + "layer": 2, + "mean_H": 1.4633128887973725, + "std_H": 0.4704755540059055, + "n": 640 + }, + { + "layer": 3, + "mean_H": 1.375873184762895, + "std_H": 0.45848071495077436, + "n": 640 + }, + { + "layer": 4, + "mean_H": 1.5902036391198635, + "std_H": 0.6494731723351617, + "n": 640 + }, + { + "layer": 5, + "mean_H": 1.7115995009051403, + "std_H": 0.7870903956688292, + "n": 640 + }, + { + "layer": 6, + "mean_H": 1.9120797269046306, + "std_H": 0.8097049457488148, + "n": 640 + }, + { + "layer": 7, + "mean_H": 1.8069396205246449, + "std_H": 0.7344557780646264, + "n": 640 + }, + { + "layer": 8, + "mean_H": 2.065459825261496, + "std_H": 0.83923774688139, + "n": 640 + }, + { + "layer": 9, + "mean_H": 2.271930713765323, + "std_H": 0.8734160192864672, + "n": 640 + }, + { + "layer": 10, + "mean_H": 2.5144276107661425, + "std_H": 0.8175109033155874, + "n": 640 + }, + { + "layer": 11, + "mean_H": 2.476173902954906, + "std_H": 0.9107822351136462, + "n": 640 + }, + { + "layer": 12, + "mean_H": 2.4304411401972175, + "std_H": 0.8737378698704705, + "n": 640 + }, + { + "layer": 13, + "mean_H": 2.2601283544674518, + "std_H": 0.8632206065111331, + "n": 640 + }, + { + "layer": 14, + "mean_H": 2.460239140409976, + "std_H": 0.8850424245790176, + "n": 640 + }, + { + "layer": 15, + "mean_H": 2.268332318495959, + "std_H": 0.8540166185749527, + "n": 640 + }, + { + "layer": 16, + "mean_H": 2.205483974982053, + "std_H": 0.8807045417722733, + "n": 640 + }, + { + "layer": 17, + "mean_H": 1.9379747526952997, + "std_H": 0.7792122230437966, + "n": 640 + }, + { + "layer": 18, + "mean_H": 1.7092706246301532, + "std_H": 0.7263209172370894, + "n": 640 + }, + { + "layer": 19, + "mean_H": 1.7558726808987557, + "std_H": 0.7262139740822701, + "n": 640 + }, + { + "layer": 20, + "mean_H": 1.8520887020975352, + "std_H": 0.8219452490570468, + "n": 640 + }, + { + "layer": 21, + "mean_H": 1.597260670363903, + "std_H": 0.7073056598392009, + "n": 640 + }, + { + "layer": 22, + "mean_H": 1.5853220832534134, + "std_H": 0.7159583922011059, + "n": 640 + }, + { + "layer": 23, + "mean_H": 1.4006114381365478, + "std_H": 0.6730863061374427, + "n": 640 + }, + { + "layer": 24, + "mean_H": 1.5882800293155015, + "std_H": 0.76919362899336, + "n": 640 + }, + { + "layer": 25, + "mean_H": 1.3493894627317786, + "std_H": 0.5917297076592941, + "n": 640 + }, + { + "layer": 26, + "mean_H": 1.6608618050813675, + "std_H": 0.9034682827427781, + "n": 640 + }, + { + "layer": 27, + "mean_H": 1.304399514105171, + "std_H": 0.5366924006154001, + "n": 640 + }, + { + "layer": 28, + "mean_H": 1.4169224985875188, + "std_H": 0.6918443950997929, + "n": 640 + }, + { + "layer": 29, + "mean_H": 1.7060872323811054, + "std_H": 0.8805211443159562, + "n": 640 + }, + { + "layer": 30, + "mean_H": 1.4526862988248468, + "std_H": 0.7414406776186819, + "n": 640 + }, + { + "layer": 31, + "mean_H": 2.2028599060606213, + "std_H": 1.0445620687662045, + "n": 640 + } + ] +} \ No newline at end of file diff --git a/data/e7_e9_hagedorn/e7_e9_hagedorn_results.json b/data/e7_e9_hagedorn/e7_e9_hagedorn_results.json new file mode 100644 index 0000000000000000000000000000000000000000..2f7fb688b4d7800393166d72e2e0528157bef240 --- /dev/null +++ b/data/e7_e9_hagedorn/e7_e9_hagedorn_results.json @@ -0,0 +1,146 @@ +{ + "n_models_e4": 16, + "scaling_law": { + "n_points": 6, + "log_a": 2.448048010691624, + "beta": -0.07898050098855318, + "R2": 0.4115137883237414, + "gamma_7B_pred": 1.930202914273422, + "models": [ + { + "model": "EleutherAI/pythia-31m", + "gamma": 1.2350013988825523, + "N_params": 31000000.0, + "chi_hagedorn": 4.255293818483924 + }, + { + "model": "EleutherAI/pythia-70m", + "gamma": 0.7476017873166874, + "N_params": 70000000.0, + "chi_hagedorn": 3.961993190715313 + }, + { + "model": "EleutherAI/pythia-410m", + "gamma": 1.0218530106365162, + "N_params": 410000000.0, + "chi_hagedorn": 45.760285236351216 + }, + { + "model": "EleutherAI/pythia-1b", + "gamma": 0.9311078627189842, + "N_params": 1000000000.0, + "chi_hagedorn": 14.515444569834292 + }, + { + "model": "EleutherAI/pythia-1.4b", + "gamma": 0.7050725013322717, + "N_params": 1400000000.0, + "chi_hagedorn": 3.3906638225234524 + }, + { + "model": "EleutherAI/pythia-2.8b", + "gamma": 0.6741618914822415, + "N_params": 2800000000.0, + "chi_hagedorn": 3.069008731204008 + } + ] + }, + "hagedorn_table": [ + { + "model": "Qwen/Qwen2.5-7B", + "gamma": 0.9966953735480816, + "chi": 302.6060629090084, + "corpus": "mongo" + }, + { + "model": "EleutherAI/pythia-14m", + "gamma": 1.003714187534367, + "chi": 269.23788601063444, + "corpus": "random" + }, + { + "model": "EleutherAI/pythia-160m", + "gamma": 1.0171452847779678, + "chi": 58.32507380017551, + "corpus": "random" + }, + { + "model": "EleutherAI/pythia-410m", + "gamma": 1.0218530106365162, + "chi": 45.760285236351216, + "corpus": "mongo" + }, + { + "model": "meta-llama/Meta-Llama-3-8B", + "gamma": 1.0454762537473639, + "chi": 21.989498201750344, + "corpus": "mongo" + }, + { + "model": "mistralai/Mistral-7B-v0.1", + "gamma": 1.060750419523944, + "chi": 16.46079167578194, + "corpus": "mongo" + }, + { + "model": "EleutherAI/pythia-1b", + "gamma": 0.9311078627189842, + "chi": 14.515444569834292, + "corpus": "mongo" + }, + { + "model": "google/gemma-2-9b-it", + "gamma": 1.1347958464287666, + "chi": 7.418626215077434, + "corpus": "random" + }, + { + "model": "mistralai/Mistral-7B-v0.1", + "gamma": 0.8296009929924347, + "chi": 5.868578799614734, + "corpus": "random" + }, + { + "model": "meta-llama/Llama-2-7b-hf", + "gamma": 0.8266242679750889, + "chi": 5.767819915282708, + "corpus": "random" + }, + { + "model": "EleutherAI/pythia-31m", + "gamma": 1.2350013988825523, + "chi": 4.255293818483924, + "corpus": "mongo" + }, + { + "model": "EleutherAI/pythia-70m", + "gamma": 0.7476017873166874, + "chi": 3.961993190715313, + "corpus": "mongo" + }, + { + "model": "EleutherAI/pythia-1.4b", + "gamma": 0.7050725013322717, + "chi": 3.3906638225234524, + "corpus": "mongo" + }, + { + "model": "EleutherAI/pythia-2.8b", + "gamma": 0.6741618914822415, + "chi": 3.069008731204008, + "corpus": "mongo" + }, + { + "model": "google/gemma-2-9b-it", + "gamma": 0.6276459084140061, + "chi": 2.6856157152473603, + "corpus": "mongo" + }, + { + "model": "meta-llama/Llama-2-7b-hf", + "gamma": 0.2870574377368437, + "chi": 1.4026375376238054, + "corpus": "mongo" + } + ] +} \ No newline at end of file diff --git a/data/e7_icl/EleutherAI--pythia-1.4b_icl.json b/data/e7_icl/EleutherAI--pythia-1.4b_icl.json new file mode 100644 index 0000000000000000000000000000000000000000..47bcc4454c1436bde4d62212d2abec77838d1169 --- /dev/null +++ b/data/e7_icl/EleutherAI--pythia-1.4b_icl.json @@ -0,0 +1,29 @@ +{ + "model": "EleutherAI/pythia-1.4b", + "gamma_obs": 0.946, + "hagedorn_dist": 0.05400000000000005, + "chi_empirical": 0.125, + "boolq_gain_0to8": 0.0, + "arc_gain_0to8": 0.25, + "boolq_by_k": { + "0": 0.7, + "1": 0.7, + "2": 0.6, + "4": 0.7, + "8": 0.7 + }, + "arc_by_k": { + "0": 0.125, + "1": 0.375, + "2": 0.25, + "4": 0.375, + "8": 0.375 + }, + "k_shots": [ + 0, + 1, + 2, + 4, + 8 + ] +} \ No newline at end of file diff --git a/data/e7_icl/EleutherAI--pythia-14m_icl.json b/data/e7_icl/EleutherAI--pythia-14m_icl.json new file mode 100644 index 0000000000000000000000000000000000000000..f2140881be9ec20ff3a8a8dbf4306f4be5e1cf09 --- /dev/null +++ b/data/e7_icl/EleutherAI--pythia-14m_icl.json @@ -0,0 +1,29 @@ +{ + "model": "EleutherAI/pythia-14m", + "gamma_obs": 1.032, + "hagedorn_dist": 0.03200000000000003, + "chi_empirical": 0.22499999999999998, + "boolq_gain_0to8": 0.19999999999999996, + "arc_gain_0to8": 0.25, + "boolq_by_k": { + "0": 0.4, + "1": 0.6, + "2": 0.5, + "4": 0.5, + "8": 0.6 + }, + "arc_by_k": { + "0": 0.0, + "1": 0.375, + "2": 0.5, + "4": 0.125, + "8": 0.25 + }, + "k_shots": [ + 0, + 1, + 2, + 4, + 8 + ] +} \ No newline at end of file diff --git a/data/e7_icl/EleutherAI--pythia-160m_icl.json b/data/e7_icl/EleutherAI--pythia-160m_icl.json new file mode 100644 index 0000000000000000000000000000000000000000..ce5ef5be72581be1e320e228605a5ced4327a52e --- /dev/null +++ b/data/e7_icl/EleutherAI--pythia-160m_icl.json @@ -0,0 +1,29 @@ +{ + "model": "EleutherAI/pythia-160m", + "gamma_obs": 1.017, + "hagedorn_dist": 0.016999999999999904, + "chi_empirical": 0.22500000000000003, + "boolq_gain_0to8": 0.20000000000000007, + "arc_gain_0to8": 0.25, + "boolq_by_k": { + "0": 0.6, + "1": 0.6, + "2": 0.5, + "4": 0.6, + "8": 0.8 + }, + "arc_by_k": { + "0": 0.0, + "1": 0.5, + "2": 0.25, + "4": 0.125, + "8": 0.25 + }, + "k_shots": [ + 0, + 1, + 2, + 4, + 8 + ] +} \ No newline at end of file diff --git a/data/e7_icl/EleutherAI--pythia-1b_icl.json b/data/e7_icl/EleutherAI--pythia-1b_icl.json new file mode 100644 index 0000000000000000000000000000000000000000..b1c6e4b1fca330c6433522d1d9809944be44b963 --- /dev/null +++ b/data/e7_icl/EleutherAI--pythia-1b_icl.json @@ -0,0 +1,29 @@ +{ + "model": "EleutherAI/pythia-1b", + "gamma_obs": 0.931, + "hagedorn_dist": 0.06899999999999995, + "chi_empirical": 0.22500000000000003, + "boolq_gain_0to8": 0.20000000000000007, + "arc_gain_0to8": 0.25, + "boolq_by_k": { + "0": 0.7, + "1": 0.7, + "2": 0.5, + "4": 0.6, + "8": 0.9 + }, + "arc_by_k": { + "0": 0.0, + "1": 0.375, + "2": 0.25, + "4": 0.125, + "8": 0.25 + }, + "k_shots": [ + 0, + 1, + 2, + 4, + 8 + ] +} \ No newline at end of file diff --git a/data/e7_icl/meta-llama--Llama-2-7b-hf_icl.json b/data/e7_icl/meta-llama--Llama-2-7b-hf_icl.json new file mode 100644 index 0000000000000000000000000000000000000000..8ddab6f7ba52a84f90f61131a620d2bd22ce6a5d --- /dev/null +++ b/data/e7_icl/meta-llama--Llama-2-7b-hf_icl.json @@ -0,0 +1,29 @@ +{ + "model": "meta-llama/Llama-2-7b-hf", + "gamma_obs": 0.783, + "hagedorn_dist": 0.21699999999999997, + "chi_empirical": 0.0, + "boolq_gain_0to8": 0.0, + "arc_gain_0to8": 0.0, + "boolq_by_k": { + "0": 0.4, + "1": 0.4, + "2": 0.6, + "4": 0.5, + "8": 0.4 + }, + "arc_by_k": { + "0": 0.0, + "1": 0.0, + "2": 0.0, + "4": 0.0, + "8": 0.0 + }, + "k_shots": [ + 0, + 1, + 2, + 4, + 8 + ] +} \ No newline at end of file diff --git a/data/e7_passkey/o_passkey_codellama.json b/data/e7_passkey/o_passkey_codellama.json new file mode 100644 index 0000000000000000000000000000000000000000..a6ecda7935c144d77a751775639d1a372eeb5f08 --- /dev/null +++ b/data/e7_passkey/o_passkey_codellama.json @@ -0,0 +1,50 @@ +{ + "model": "codellama", + "l_crit": 28, + "rope_theta": 1000000, + "n_layers": 40, + "n_prompts": 50, + "passkey_digits": 5, + "distances": { + "100": { + "exact_match_base": 0.3, + "exact_match_transplant": 0.3, + "digit_overlap_base": 0.43200000000000005, + "digit_overlap_transplant": 0.436, + "improvement_exact": 0.0, + "improvement_digit": 0.003999999999999948 + }, + "200": { + "exact_match_base": 0.44, + "exact_match_transplant": 0.46, + "digit_overlap_base": 0.5680000000000001, + "digit_overlap_transplant": 0.584, + "improvement_exact": 0.020000000000000018, + "improvement_digit": 0.015999999999999903 + }, + "500": { + "exact_match_base": 0.22, + "exact_match_transplant": 0.2, + "digit_overlap_base": 0.244, + "digit_overlap_transplant": 0.23199999999999998, + "improvement_exact": -0.01999999999999999, + "improvement_digit": -0.01200000000000001 + }, + "1000": { + "exact_match_base": 0.16, + "exact_match_transplant": 0.16, + "digit_overlap_base": 0.21200000000000002, + "digit_overlap_transplant": 0.21200000000000002, + "improvement_exact": 0.0, + "improvement_digit": 0.0 + }, + "2000": { + "exact_match_base": 0.04, + "exact_match_transplant": 0.04, + "digit_overlap_base": 0.09200000000000001, + "digit_overlap_transplant": 0.09200000000000001, + "improvement_exact": 0.0, + "improvement_digit": 0.0 + } + } +} \ No newline at end of file diff --git a/data/e7_passkey/o_passkey_mistral.json b/data/e7_passkey/o_passkey_mistral.json new file mode 100644 index 0000000000000000000000000000000000000000..d458113786d6250dcb566f6973ee35e38726e414 --- /dev/null +++ b/data/e7_passkey/o_passkey_mistral.json @@ -0,0 +1,50 @@ +{ + "model": "mistral", + "l_crit": 31, + "rope_theta": 10000, + "n_layers": 32, + "n_prompts": 50, + "passkey_digits": 5, + "distances": { + "100": { + "exact_match_base": 0.02, + "exact_match_transplant": 0.02, + "digit_overlap_base": 0.02, + "digit_overlap_transplant": 0.02, + "improvement_exact": 0.0, + "improvement_digit": 0.0 + }, + "200": { + "exact_match_base": 0.02, + "exact_match_transplant": 0.02, + "digit_overlap_base": 0.02, + "digit_overlap_transplant": 0.02, + "improvement_exact": 0.0, + "improvement_digit": 0.0 + }, + "500": { + "exact_match_base": 0.0, + "exact_match_transplant": 0.0, + "digit_overlap_base": 0.0, + "digit_overlap_transplant": 0.0, + "improvement_exact": 0.0, + "improvement_digit": 0.0 + }, + "1000": { + "exact_match_base": 0.0, + "exact_match_transplant": 0.0, + "digit_overlap_base": 0.0, + "digit_overlap_transplant": 0.0, + "improvement_exact": 0.0, + "improvement_digit": 0.0 + }, + "2000": { + "exact_match_base": 0.0, + "exact_match_transplant": 0.0, + "digit_overlap_base": 0.0, + "digit_overlap_transplant": 0.0, + "improvement_exact": 0.0, + "improvement_digit": 0.0 + } + } +} \ No newline at end of file diff --git a/data/e7_passkey/o_passkey_summary.json b/data/e7_passkey/o_passkey_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..52a101916cd7e7dcf4ed38fa39095b0759787779 --- /dev/null +++ b/data/e7_passkey/o_passkey_summary.json @@ -0,0 +1,16 @@ +{ + "codellama": { + "exact_match_at_d500_base": 0.22, + "exact_match_at_d500_transplant": 0.2, + "exact_match_at_d2000_base": 0.04, + "exact_match_at_d2000_transplant": 0.04, + "max_improvement_exact": 0.020000000000000018 + }, + "mistral": { + "exact_match_at_d500_base": 0.0, + "exact_match_at_d500_transplant": 0.0, + "exact_match_at_d2000_base": 0.0, + "exact_match_at_d2000_transplant": 0.0, + "max_improvement_exact": 0.0 + } +} \ No newline at end of file diff --git a/data/e_goldstone/EleutherAI--pythia-160m.json b/data/e_goldstone/EleutherAI--pythia-160m.json new file mode 100644 index 0000000000000000000000000000000000000000..f36f69a1849a621cb4f912920537cb042dba2bb6 --- /dev/null +++ b/data/e_goldstone/EleutherAI--pythia-160m.json @@ -0,0 +1,50 @@ +{ + "model": "EleutherAI/pythia-160m", + "n_layers": 12, + "n_heads": 12, + "seq_len": 512, + "n_samples": 50, + "l_crit_from_e1": { + "mean": 3.5, + "values": [ + 3, + 4 + ] + }, + "peak_cv_layer_1indexed": 4, + "peak_cv_value": 0.000647, + "min_entropy_layer_1indexed": 4, + "min_entropy_value": 2.4524, + "variance_per_layer": [ + 0.000198, + 0.00029, + 0.00024, + 0.000647, + 0.000553, + 0.000437, + 0.0004, + 0.000597, + 0.000407, + 0.000412, + 9.4e-05, + 0.000223 + ], + "entropy_per_layer": [ + 3.8929, + 3.4553, + 3.6503, + 2.4524, + 2.5338, + 3.1658, + 3.6614, + 3.0596, + 3.836, + 3.8576, + 5.3251, + 4.6401 + ], + "n_nan_layers": 0, + "dtype": "bfloat16", + "decision": "CONFIRMED: peak_Cv=4 \u2248 L_crit=3.5 (\u0394=0.5)", + "runtime_seconds": 1.9 +} \ No newline at end of file diff --git a/data/e_goldstone/EleutherAI--pythia-1b.json b/data/e_goldstone/EleutherAI--pythia-1b.json new file mode 100644 index 0000000000000000000000000000000000000000..c828649c3c471dcbf5c530a765d200249746263f --- /dev/null +++ b/data/e_goldstone/EleutherAI--pythia-1b.json @@ -0,0 +1,58 @@ +{ + "model": "EleutherAI/pythia-1b", + "n_layers": 16, + "n_heads": 8, + "seq_len": 512, + "n_samples": 50, + "l_crit_from_e1": { + "mean": 15.0, + "values": [ + 15, + 15 + ] + }, + "peak_cv_layer_1indexed": 11, + "peak_cv_value": 0.000822, + "min_entropy_layer_1indexed": 11, + "min_entropy_value": 2.265, + "variance_per_layer": [ + 6.2e-05, + 0.000159, + 0.000155, + 0.00035, + 0.00031, + 0.000318, + 0.000495, + 0.000661, + 0.000655, + 0.000516, + 0.000822, + 0.000324, + 0.000533, + 0.000486, + 0.00013, + 0.000284 + ], + "entropy_per_layer": [ + 4.5899, + 4.0317, + 4.2177, + 3.9346, + 3.5781, + 3.4876, + 3.0741, + 2.438, + 3.0571, + 3.3015, + 2.265, + 3.9201, + 3.2108, + 3.3708, + 4.6141, + 3.5329 + ], + "n_nan_layers": 0, + "dtype": "bfloat16", + "decision": "NEAR: peak_Cv=11, L_crit=15.0 (\u0394=4.0)", + "runtime_seconds": 5.9 +} \ No newline at end of file diff --git a/data/e_goldstone/EleutherAI--pythia-2.8b.json b/data/e_goldstone/EleutherAI--pythia-2.8b.json new file mode 100644 index 0000000000000000000000000000000000000000..2a202a9130389c1c0e9b59199391a3be87454889 --- /dev/null +++ b/data/e_goldstone/EleutherAI--pythia-2.8b.json @@ -0,0 +1,84 @@ +{ + "model": "EleutherAI/pythia-2.8b", + "n_layers": 32, + "n_heads": 32, + "seq_len": 512, + "n_samples": 50, + "l_crit_from_e1": null, + "peak_cv_layer_1indexed": 32, + "peak_cv_value": 0.001393, + "min_entropy_layer_1indexed": 32, + "min_entropy_value": 0.7427, + "variance_per_layer": [ + 7.1e-05, + 0.000323, + 0.000101, + 0.000582, + 0.000441, + 0.00053, + 0.000482, + 0.000707, + 0.000801, + 0.000689, + 0.000744, + 0.000653, + 0.000583, + 0.000679, + 0.001011, + 0.000789, + 0.000789, + 0.000775, + 0.000752, + 0.000853, + 0.000786, + 0.000742, + 0.00071, + 0.000693, + 0.000569, + 0.000657, + 0.000642, + 0.000637, + 0.000616, + 0.000749, + 0.000836, + 0.001393 + ], + "entropy_per_layer": [ + 4.7854, + 3.5542, + 4.8828, + 2.8463, + 3.2198, + 2.9146, + 3.3088, + 2.5581, + 2.2702, + 2.5544, + 2.3549, + 2.9783, + 3.0359, + 2.8657, + 1.8431, + 2.4528, + 2.3897, + 2.4973, + 2.5624, + 2.2672, + 2.3972, + 2.3401, + 2.4305, + 2.4288, + 2.6802, + 2.4344, + 2.4846, + 2.3994, + 2.5019, + 1.9551, + 1.7849, + 0.7427 + ], + "n_nan_layers": 0, + "dtype": "bfloat16", + "decision": "L_crit unknown (E1 not run)", + "runtime_seconds": 8.3 +} \ No newline at end of file diff --git a/data/e_goldstone/EleutherAI--pythia-410m.json b/data/e_goldstone/EleutherAI--pythia-410m.json new file mode 100644 index 0000000000000000000000000000000000000000..a924587ba7f04e2b9a1acbb606ce2b75383da8e1 --- /dev/null +++ b/data/e_goldstone/EleutherAI--pythia-410m.json @@ -0,0 +1,68 @@ +{ + "model": "EleutherAI/pythia-410m", + "n_layers": 24, + "n_heads": 16, + "seq_len": 512, + "n_samples": 50, + "l_crit_from_e1": null, + "peak_cv_layer_1indexed": 15, + "peak_cv_value": 0.001081, + "min_entropy_layer_1indexed": 15, + "min_entropy_value": 1.768, + "variance_per_layer": [ + 0.000118, + 0.000258, + 7e-05, + 0.00018, + 0.000141, + 0.000434, + 0.00049, + 0.000727, + 0.000499, + 0.000619, + 0.000542, + 0.000904, + 0.000716, + 0.000971, + 0.001081, + 0.000855, + 0.000505, + 0.000708, + 0.00057, + 0.000663, + 0.000236, + 0.000163, + 0.00015, + 0.000142 + ], + "entropy_per_layer": [ + 4.3774, + 3.536, + 4.8356, + 4.1821, + 4.5478, + 2.9041, + 3.0163, + 2.4139, + 2.8523, + 2.7225, + 3.0353, + 2.2303, + 2.6803, + 1.9203, + 1.768, + 2.1956, + 3.2916, + 2.7684, + 3.3229, + 3.3017, + 4.7612, + 5.0086, + 5.0369, + 5.1515 + ], + "n_nan_layers": 0, + "dtype": "bfloat16", + "decision": "L_crit unknown (E1 not run)", + "runtime_seconds": 4.9 +} \ No newline at end of file diff --git a/data/e_goldstone/EleutherAI--pythia-70m.json b/data/e_goldstone/EleutherAI--pythia-70m.json new file mode 100644 index 0000000000000000000000000000000000000000..0d8384cf69f54f9f01da9bf2b6612bc00c10406b --- /dev/null +++ b/data/e_goldstone/EleutherAI--pythia-70m.json @@ -0,0 +1,39 @@ +{ + "model": "EleutherAI/pythia-70m", + "n_layers": 6, + "n_heads": 8, + "seq_len": 512, + "n_samples": 50, + "l_crit_from_e1": { + "mean": 4.0, + "values": [ + 4, + 4, + 4 + ] + }, + "peak_cv_layer_1indexed": 2, + "peak_cv_value": 0.000368, + "min_entropy_layer_1indexed": 2, + "min_entropy_value": 2.9956, + "variance_per_layer": [ + 0.000267, + 0.000368, + 0.000364, + 8e-05, + 6.9e-05, + 9.3e-05 + ], + "entropy_per_layer": [ + 3.4305, + 2.9956, + 3.2717, + 5.0246, + 5.4477, + 5.3929 + ], + "n_nan_layers": 0, + "dtype": "bfloat16", + "decision": "CONFIRMED: peak_Cv=2 \u2248 L_crit=4.0 (\u0394=2.0)", + "runtime_seconds": 1.2 +} \ No newline at end of file diff --git a/data/e_goldstone/Qwen--Qwen2.5-7B.json b/data/e_goldstone/Qwen--Qwen2.5-7B.json new file mode 100644 index 0000000000000000000000000000000000000000..b5171b02b8b7963ac576d802c9168be8aa3fba10 --- /dev/null +++ b/data/e_goldstone/Qwen--Qwen2.5-7B.json @@ -0,0 +1,76 @@ +{ + "model": "Qwen/Qwen2.5-7B", + "n_layers": 28, + "n_heads": 28, + "seq_len": 512, + "n_samples": 50, + "l_crit_from_e1": null, + "peak_cv_layer_1indexed": 5, + "peak_cv_value": 0.000913, + "min_entropy_layer_1indexed": 11, + "min_entropy_value": 2.0645, + "variance_per_layer": [ + 0.000439, + 0.000216, + 0.000115, + 0.000241, + 0.000913, + 0.000626, + 0.000801, + 0.000508, + 0.000745, + 0.000349, + 0.000821, + 0.000669, + 0.000579, + 0.000473, + 0.000678, + 0.000626, + 0.000427, + 0.00034, + 0.000661, + 0.000602, + 0.00062, + 0.000531, + 0.000799, + 0.000601, + 0.000558, + 0.0007, + 0.000602, + 0.000678 + ], + "entropy_per_layer": [ + 3.4547, + 4.2553, + 4.4153, + 3.8952, + 2.2419, + 2.9846, + 2.2191, + 3.1387, + 2.4572, + 3.8687, + 2.0645, + 2.4215, + 2.9864, + 3.123, + 2.8327, + 2.3509, + 3.312, + 3.7269, + 2.8109, + 3.1786, + 3.2028, + 3.209, + 2.8174, + 2.9818, + 3.3167, + 2.8269, + 2.9339, + 2.7591 + ], + "n_nan_layers": 0, + "dtype": "bfloat16", + "decision": "L_crit unknown (E1 not run)", + "runtime_seconds": 208.9 +} \ No newline at end of file diff --git a/data/e_goldstone/gpt2.json b/data/e_goldstone/gpt2.json new file mode 100644 index 0000000000000000000000000000000000000000..31de7d109f2aa3a835352503fb962eef2bd16529 --- /dev/null +++ b/data/e_goldstone/gpt2.json @@ -0,0 +1,44 @@ +{ + "model": "gpt2", + "n_layers": 12, + "n_heads": 12, + "seq_len": 512, + "n_samples": 50, + "l_crit_from_e1": null, + "peak_cv_layer_1indexed": 5, + "peak_cv_value": 0.001014, + "min_entropy_layer_1indexed": 5, + "min_entropy_value": 1.5913, + "variance_per_layer": [ + 0.000636, + 0.000181, + 0.000422, + 0.0006, + 0.001014, + 0.000986, + 0.000806, + 0.000947, + 0.001006, + 0.000882, + 0.000578, + 0.000485 + ], + "entropy_per_layer": [ + 3.2678, + 4.3383, + 2.4266, + 2.0365, + 1.5913, + 1.8568, + 2.2142, + 2.4585, + 2.0083, + 2.3891, + 3.2117, + 3.546 + ], + "n_nan_layers": 0, + "dtype": "bfloat16", + "decision": "L_crit unknown (E1 not run)", + "runtime_seconds": 1.1 +} \ No newline at end of file diff --git a/data/e_goldstone/meta-llama--Meta-Llama-3-8B.json b/data/e_goldstone/meta-llama--Meta-Llama-3-8B.json new file mode 100644 index 0000000000000000000000000000000000000000..111b81c6f506f51bcdf58e414c78ce7f12c3d4f1 --- /dev/null +++ b/data/e_goldstone/meta-llama--Meta-Llama-3-8B.json @@ -0,0 +1,84 @@ +{ + "model": "meta-llama/Meta-Llama-3-8B", + "n_layers": 32, + "n_heads": 32, + "seq_len": 512, + "n_samples": 50, + "l_crit_from_e1": null, + "peak_cv_layer_1indexed": 19, + "peak_cv_value": 0.001493, + "min_entropy_layer_1indexed": 19, + "min_entropy_value": 1.0118, + "variance_per_layer": [ + 0.000145, + 0.000375, + 0.001389, + 0.001226, + 0.001048, + 0.001033, + 0.000899, + 0.000841, + 0.000798, + 0.000699, + 0.000709, + 0.000571, + 0.000837, + 0.000704, + 0.000761, + 0.000595, + 0.0008, + 0.000771, + 0.001493, + 0.001003, + 0.001279, + 0.001108, + 0.00109, + 0.001337, + 0.001339, + 0.00117, + 0.000881, + 0.001137, + 0.001006, + 0.001054, + 0.000596, + 0.000735 + ], + "entropy_per_layer": [ + 4.6611, + 3.4838, + 1.2565, + 1.625, + 1.8219, + 1.8352, + 2.1577, + 2.2191, + 2.3617, + 2.7301, + 2.6664, + 3.0819, + 2.1025, + 2.9503, + 2.5696, + 3.2183, + 2.4381, + 2.7608, + 1.0118, + 2.2512, + 1.4995, + 1.8092, + 1.9047, + 1.3375, + 1.2615, + 1.5836, + 2.4054, + 1.8537, + 2.0616, + 2.0781, + 3.2291, + 2.6932 + ], + "n_nan_layers": 0, + "dtype": "bfloat16", + "decision": "L_crit unknown (E1 not run)", + "runtime_seconds": 2335.2 +} \ No newline at end of file diff --git a/data/e_task_binding/gpt2-large.json b/data/e_task_binding/gpt2-large.json new file mode 100644 index 0000000000000000000000000000000000000000..75247cc79e8491419e8a3816e8236868b014f541 --- /dev/null +++ b/data/e_task_binding/gpt2-large.json @@ -0,0 +1,54 @@ +{ + "model": "gpt2-large", + "N": 36, + "L_crit": 33, + "N_sem": 3, + "tipo": "III", + "acc_d1": 0.0, + "d_half": 1, + "by_distance": { + "1": { + "acc": 0.0, + "gap": 0.0, + "n": 30 + }, + "5": { + "acc": 0.0, + "gap": 0.26210028330485025, + "n": 30 + }, + "10": { + "acc": 0.0, + "gap": 0.3697531779607137, + "n": 30 + }, + "20": { + "acc": 0.0, + "gap": 0.3226956685384115, + "n": 30 + }, + "50": { + "acc": 0.0, + "gap": 0.6995176315307617, + "n": 30 + }, + "100": { + "acc": 0.03333333333333333, + "gap": 1.1318510214487711, + "n": 30 + }, + "200": { + "acc": 0.0, + "gap": 1.0689163525899252, + "n": 30 + }, + "500": { + "acc": 0.0, + "gap": 1.3151949564615886, + "n": 30 + } + }, + "d_half_gap": 50, + "max_gap": 1.3151949564615886, + "note_acc": "top-1 acc unusable (baseline 1/30); use gap metric" +} \ No newline at end of file diff --git a/data/e_task_binding/gpt2-medium.json b/data/e_task_binding/gpt2-medium.json new file mode 100644 index 0000000000000000000000000000000000000000..af4f8d9ae8022ea364a347fb4fa5677b8d6eef28 --- /dev/null +++ b/data/e_task_binding/gpt2-medium.json @@ -0,0 +1,54 @@ +{ + "model": "gpt2-medium", + "N": 24, + "L_crit": 23, + "N_sem": 1, + "tipo": "I", + "acc_d1": 0.03333333333333333, + "d_half": 10, + "by_distance": { + "1": { + "acc": 0.03333333333333333, + "gap": 0.0, + "n": 30 + }, + "5": { + "acc": 0.03333333333333333, + "gap": 0.45460240840911864, + "n": 30 + }, + "10": { + "acc": 0.0, + "gap": 0.6843744039535522, + "n": 30 + }, + "20": { + "acc": 0.03333333333333333, + "gap": 0.597429347038269, + "n": 30 + }, + "50": { + "acc": 0.03333333333333333, + "gap": 0.7374829371770223, + "n": 30 + }, + "100": { + "acc": 0.03333333333333333, + "gap": 1.0384546836217246, + "n": 30 + }, + "200": { + "acc": 0.03333333333333333, + "gap": 1.0752163330713909, + "n": 30 + }, + "500": { + "acc": 0.03333333333333333, + "gap": 1.047822388013204, + "n": 30 + } + }, + "d_half_gap": 10, + "max_gap": 1.0752163330713909, + "note_acc": "top-1 acc unusable (baseline 1/30); use gap metric" +} \ No newline at end of file diff --git a/data/e_task_binding/gpt2-xl.json b/data/e_task_binding/gpt2-xl.json new file mode 100644 index 0000000000000000000000000000000000000000..11aedb7e9488805675a048a5e2a3751330cc66b2 --- /dev/null +++ b/data/e_task_binding/gpt2-xl.json @@ -0,0 +1,54 @@ +{ + "model": "gpt2-xl", + "N": 48, + "L_crit": 43, + "N_sem": 5, + "tipo": "II", + "acc_d1": 0.0, + "d_half": 1, + "by_distance": { + "1": { + "acc": 0.0, + "gap": 0.0, + "n": 30 + }, + "5": { + "acc": 0.0, + "gap": 0.48437228202819826, + "n": 30 + }, + "10": { + "acc": 0.0, + "gap": 0.7611155986785889, + "n": 30 + }, + "20": { + "acc": 0.0, + "gap": 0.8163606325785319, + "n": 30 + }, + "50": { + "acc": 0.03333333333333333, + "gap": 0.925626007715861, + "n": 30 + }, + "100": { + "acc": 0.03333333333333333, + "gap": 1.1545397281646728, + "n": 30 + }, + "200": { + "acc": 0.03333333333333333, + "gap": 1.1601056575775146, + "n": 30 + }, + "500": { + "acc": 0.0, + "gap": 1.2979705969492594, + "n": 30 + } + }, + "d_half_gap": 10, + "max_gap": 1.2979705969492594, + "note_acc": "top-1 acc unusable (baseline 1/30); use gap metric" +} \ No newline at end of file diff --git a/data/e_task_binding/gpt2.json b/data/e_task_binding/gpt2.json new file mode 100644 index 0000000000000000000000000000000000000000..4e8dd1581a875f2ac97a7815513df3eb5c21bb3a --- /dev/null +++ b/data/e_task_binding/gpt2.json @@ -0,0 +1,54 @@ +{ + "model": "gpt2", + "N": 12, + "L_crit": 11, + "N_sem": 1, + "tipo": "I", + "acc_d1": 0.03333333333333333, + "d_half": 20, + "by_distance": { + "1": { + "acc": 0.03333333333333333, + "gap": 0.0, + "n": 30 + }, + "5": { + "acc": 0.03333333333333333, + "gap": 0.436801815032959, + "n": 30 + }, + "10": { + "acc": 0.03333333333333333, + "gap": 0.7799602349599203, + "n": 30 + }, + "20": { + "acc": 0.0, + "gap": 0.6915457248687744, + "n": 30 + }, + "50": { + "acc": 0.0, + "gap": 0.7360549132029216, + "n": 30 + }, + "100": { + "acc": 0.0, + "gap": 0.8788070201873779, + "n": 30 + }, + "200": { + "acc": 0.03333333333333333, + "gap": 0.8997215747833252, + "n": 30 + }, + "500": { + "acc": 0.0, + "gap": 0.7841876029968262, + "n": 30 + } + }, + "d_half_gap": 10, + "max_gap": 0.8997215747833252, + "note_acc": "top-1 acc unusable (baseline 1/30); use gap metric" +} \ No newline at end of file diff --git a/data/exp_alibi_extended/bloom_1b7_alibi_extended.json b/data/exp_alibi_extended/bloom_1b7_alibi_extended.json new file mode 100644 index 0000000000000000000000000000000000000000..1b9da396409999ba461091c4793387aa72b66644 --- /dev/null +++ b/data/exp_alibi_extended/bloom_1b7_alibi_extended.json @@ -0,0 +1,50 @@ +{ + "model": "bigscience/bloom-1b7", + "T_train": 2048, + "distances": [ + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2048, + 3000, + 5000 + ], + "means": [ + 0.014610571331448026, + 0.007812359597947863, + 0.003867449363072713, + 0.0014175668358802794, + 0.0008336269193225436, + 0.0001911761032210456, + 0.0003879707720544604, + 2.571443716684977e-06, + 9.901821613311768e-06, + 0.0 + ], + "stds": { + "10": 0.004878007416589143, + "20": 0.002834609612370771, + "50": 0.0023298736010873714, + "100": 0.0005476547332398899, + "200": 0.0006483472385705435, + "500": 0.00012947732223353835, + "1000": 0.0007673620877808975, + "2048": 3.4976826887877778e-06, + "3000": 9.889403978983562e-06, + "5000": 0.0 + }, + "gamma_short": 1.3155946562780965, + "r2_short": 0.8239702509190621, + "lambda_ext": NaN, + "r2_exponential": NaN, + "local_gammas": { + "1000->2048": 6.997791153546874, + "2048->3000": -3.531778003334585, + "3000->5000": Infinity + }, + "alibi_collapse_confirmed": true +} \ No newline at end of file diff --git a/data/exp_b1/niah_alpha_opt_results.json b/data/exp_b1/niah_alpha_opt_results.json new file mode 100644 index 0000000000000000000000000000000000000000..a2bfe3bbbcd3fd975eeb335e55a1cfb62bc490b0 --- /dev/null +++ b/data/exp_b1/niah_alpha_opt_results.json @@ -0,0 +1,159 @@ +{ + "model": "EleutherAI/pythia-1b", + "gamma_text": 0.931, + "gamma_target": 0.5, + "alpha_opt_theory": 10000.000000000018, + "C": 9.210340371976184, + "results": { + "alpha_1": { + "alpha": 1.0, + "results": { + "512": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "1024": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "2048": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "3072": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "4096": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + } + } + }, + "alpha_4": { + "alpha": 4.0, + "results": { + "512": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "1024": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "2048": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "3072": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "4096": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + } + } + }, + "alpha_8": { + "alpha": 8.0, + "results": { + "512": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "1024": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "2048": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "3072": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "4096": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + } + } + }, + "alpha_16": { + "alpha": 16.0, + "results": { + "512": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "1024": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "2048": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "3072": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "4096": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + } + } + }, + "alpha_opt": { + "alpha": 64.0, + "results": { + "512": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "1024": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "2048": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "3072": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + }, + "4096": { + "accuracy": 0.0, + "correct": 0, + "total": 40 + } + } + } + } +} \ No newline at end of file diff --git a/data/exp_b1/niah_v4_logprob_results.json b/data/exp_b1/niah_v4_logprob_results.json new file mode 100644 index 0000000000000000000000000000000000000000..3827f0fc08c0464025f91b67d0d3c5abb836fab2 --- /dev/null +++ b/data/exp_b1/niah_v4_logprob_results.json @@ -0,0 +1,151 @@ +{ + "model": "EleutherAI/pythia-70m", + "theta_base": 10000, + "T_train": 2048, + "gamma_obs": 0.748, + "N_trials": 50, + "N_wrong": 9, + "chance_level": 0.1, + "results": { + "alpha_1": { + "alpha": 1.0, + "results": { + "512": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 0.92, + "correct": 46, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 0.92, + "correct": 46, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.1, + "correct": 5, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_4": { + "alpha": 4.0, + "results": { + "512": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 0.9, + "correct": 45, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 0.9, + "correct": 45, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.86, + "correct": 43, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_8": { + "alpha": 8.0, + "results": { + "512": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 0.84, + "correct": 42, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_16": { + "alpha": 16.0, + "results": { + "512": { + "accuracy": 0.74, + "correct": 37, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 0.84, + "correct": 42, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 0.82, + "correct": 41, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.82, + "correct": 41, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.84, + "correct": 42, + "total": 50, + "chance": 0.1 + } + } + } + } +} \ No newline at end of file diff --git a/data/exp_b1/niah_v5_extended.json b/data/exp_b1/niah_v5_extended.json new file mode 100644 index 0000000000000000000000000000000000000000..1f106fa97a39fd08001af7d8ef8cdbe5fddddfe1 --- /dev/null +++ b/data/exp_b1/niah_v5_extended.json @@ -0,0 +1,466 @@ +{ + "config": { + "N_trials": 50, + "N_wrong": 9, + "chance": 0.1, + "lengths": [ + 512, + 1024, + 2048, + 3072, + 4096 + ], + "alphas": { + "alpha_1": 1.0, + "alpha_4": 4.0, + "alpha_8": 8.0, + "alpha_16": 16.0 + } + }, + "models": { + "pythia-70m": { + "model": "EleutherAI/pythia-70m", + "gamma": 0.748, + "theta_base": 10000, + "T_train": 2048, + "alphas": { + "alpha_1": { + "alpha": 1.0, + "results": { + "512": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 0.92, + "correct": 46, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 0.92, + "correct": 46, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.1, + "correct": 5, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_4": { + "alpha": 4.0, + "results": { + "512": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 0.9, + "correct": 45, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 0.9, + "correct": 45, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.86, + "correct": 43, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_8": { + "alpha": 8.0, + "results": { + "512": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 0.84, + "correct": 42, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_16": { + "alpha": 16.0, + "results": { + "512": { + "accuracy": 0.74, + "correct": 37, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 0.84, + "correct": 42, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 0.82, + "correct": 41, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.82, + "correct": 41, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.84, + "correct": 42, + "total": 50, + "chance": 0.1 + } + } + } + } + }, + "pythia-1b": { + "model": "EleutherAI/pythia-1b", + "gamma": 0.931, + "theta_base": 10000, + "T_train": 2048, + "alphas": { + "alpha_1": { + "alpha": 1.0, + "results": { + "512": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.16, + "correct": 8, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.16, + "correct": 8, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_4": { + "alpha": 4.0, + "results": { + "512": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_8": { + "alpha": 8.0, + "results": { + "512": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_16": { + "alpha": 16.0, + "results": { + "512": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 0.92, + "correct": 46, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.9, + "correct": 45, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.88, + "correct": 44, + "total": 50, + "chance": 0.1 + } + } + } + } + }, + "pythia-1.4b": { + "model": "EleutherAI/pythia-1.4b", + "gamma": 0.705, + "theta_base": 10000, + "T_train": 2048, + "alphas": { + "alpha_1": { + "alpha": 1.0, + "results": { + "512": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.16, + "correct": 8, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.12, + "correct": 6, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_4": { + "alpha": 4.0, + "results": { + "512": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_8": { + "alpha": 8.0, + "results": { + "512": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + } + } + }, + "alpha_16": { + "alpha": 16.0, + "results": { + "512": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "1024": { + "accuracy": 1.0, + "correct": 50, + "total": 50, + "chance": 0.1 + }, + "2048": { + "accuracy": 0.98, + "correct": 49, + "total": 50, + "chance": 0.1 + }, + "3072": { + "accuracy": 0.98, + "correct": 49, + "total": 50, + "chance": 0.1 + }, + "4096": { + "accuracy": 0.98, + "correct": 49, + "total": 50, + "chance": 0.1 + } + } + } + } + } + } +} \ No newline at end of file diff --git a/data/exp_b1/ntk_longseq_results.json b/data/exp_b1/ntk_longseq_results.json new file mode 100644 index 0000000000000000000000000000000000000000..ae70fa57a26132067c5692e9700f620035d79be4 --- /dev/null +++ b/data/exp_b1/ntk_longseq_results.json @@ -0,0 +1,142 @@ +{ + "model": "EleutherAI/pythia-1b", + "seq_len": 4096, + "results": { + "alpha_1": { + "alpha": 1.0, + "theta_new": 10000, + "bands": { + "A_in_train": { + "range": [ + 960, + 1024 + ], + "ppl": 11.94919679725793 + }, + "B_boundary": { + "range": [ + 1984, + 2048 + ], + "ppl": 12.659132687210402 + }, + "C_1p5x": { + "range": [ + 3008, + 3072 + ], + "ppl": 620.9496501605687 + }, + "D_2x": { + "range": [ + 4032, + 4096 + ], + "ppl": 1424.1467265906977 + } + } + }, + "alpha_4": { + "alpha": 4.0, + "theta_new": 40000, + "bands": { + "A_in_train": { + "range": [ + 960, + 1024 + ], + "ppl": 13.877564029792271 + }, + "B_boundary": { + "range": [ + 1984, + 2048 + ], + "ppl": 14.549244421859534 + }, + "C_1p5x": { + "range": [ + 3008, + 3072 + ], + "ppl": 16.084113267384442 + }, + "D_2x": { + "range": [ + 4032, + 4096 + ], + "ppl": 12.365884194357319 + } + } + }, + "alpha_16": { + "alpha": 16.0, + "theta_new": 160000, + "bands": { + "A_in_train": { + "range": [ + 960, + 1024 + ], + "ppl": 40.116867227997055 + }, + "B_boundary": { + "range": [ + 1984, + 2048 + ], + "ppl": 59.7889171894188 + }, + "C_1p5x": { + "range": [ + 3008, + 3072 + ], + "ppl": 72.9693188079783 + }, + "D_2x": { + "range": [ + 4032, + 4096 + ], + "ppl": 77.5390166251303 + } + } + }, + "alpha_opt": { + "alpha": 64.0, + "theta_new": 640000, + "bands": { + "A_in_train": { + "range": [ + 960, + 1024 + ], + "ppl": 79.96912518546141 + }, + "B_boundary": { + "range": [ + 1984, + 2048 + ], + "ppl": 143.51021968773875 + }, + "C_1p5x": { + "range": [ + 3008, + 3072 + ], + "ppl": 172.83610058973207 + }, + "D_2x": { + "range": [ + 4032, + 4096 + ], + "ppl": 219.85511253338788 + } + } + } + } +} \ No newline at end of file diff --git a/data/exp_b1/ntk_ppl_results.json b/data/exp_b1/ntk_ppl_results.json new file mode 100644 index 0000000000000000000000000000000000000000..13b81d691b49dd74a17229d9787991729e2c0101 --- /dev/null +++ b/data/exp_b1/ntk_ppl_results.json @@ -0,0 +1,139 @@ +{ + "model": "EleutherAI/pythia-1b", + "gamma_text": 0.931, + "gamma_target": 0.5, + "alpha_opt_theory": 64.0, + "theta_base": 10000, + "results": { + "alpha_1": { + "alpha": 1.0, + "theta_new": 10000, + "depths": { + "in_train": { + "ctx_end": 1024, + "ctx_len": 124, + "ppl": 17.404948412148393 + }, + "at_boundary": { + "ctx_end": 2048, + "ctx_len": 148, + "ppl": 15.854351582229326 + }, + "out_1.5x": { + "ctx_end": 2650, + "ctx_len": 150, + "ppl": 16.97577129243789 + }, + "out_2x": { + "ctx_end": 3650, + "ctx_len": 150, + "ppl": 15.524942384458516 + } + } + }, + "alpha_4": { + "alpha": 4.0, + "theta_new": 40000, + "depths": { + "in_train": { + "ctx_end": 1024, + "ctx_len": 124, + "ppl": 18.491019027713858 + }, + "at_boundary": { + "ctx_end": 2048, + "ctx_len": 148, + "ppl": 16.854908619134363 + }, + "out_1.5x": { + "ctx_end": 2650, + "ctx_len": 150, + "ppl": 18.12599335819514 + }, + "out_2x": { + "ctx_end": 3650, + "ctx_len": 150, + "ppl": 16.456427754310326 + } + } + }, + "alpha_8": { + "alpha": 8.0, + "theta_new": 80000, + "depths": { + "in_train": { + "ctx_end": 1024, + "ctx_len": 124, + "ppl": 19.732665122735888 + }, + "at_boundary": { + "ctx_end": 2048, + "ctx_len": 148, + "ppl": 18.26191435396263 + }, + "out_1.5x": { + "ctx_end": 2650, + "ctx_len": 150, + "ppl": 19.605890918116287 + }, + "out_2x": { + "ctx_end": 3650, + "ctx_len": 150, + "ppl": 17.594068865300173 + } + } + }, + "alpha_16": { + "alpha": 16.0, + "theta_new": 160000, + "depths": { + "in_train": { + "ctx_end": 1024, + "ctx_len": 124, + "ppl": 21.76860403814333 + }, + "at_boundary": { + "ctx_end": 2048, + "ctx_len": 148, + "ppl": 20.56687230107969 + }, + "out_1.5x": { + "ctx_end": 2650, + "ctx_len": 150, + "ppl": 21.782426386202566 + }, + "out_2x": { + "ctx_end": 3650, + "ctx_len": 150, + "ppl": 19.421047912281473 + } + } + }, + "alpha_opt": { + "alpha": 64.0, + "theta_new": 640000, + "depths": { + "in_train": { + "ctx_end": 1024, + "ctx_len": 124, + "ppl": 30.34100202780192 + }, + "at_boundary": { + "ctx_end": 2048, + "ctx_len": 148, + "ppl": 31.045355589390898 + }, + "out_1.5x": { + "ctx_end": 2650, + "ctx_len": 150, + "ppl": 31.12833474923277 + }, + "out_2x": { + "ctx_end": 3650, + "ctx_len": 150, + "ppl": 27.826159362193188 + } + } + } + } +} \ No newline at end of file diff --git a/data/exp_b2/kv_compression_extended.json b/data/exp_b2/kv_compression_extended.json new file mode 100644 index 0000000000000000000000000000000000000000..917c799877944b3bc237967492a9b5afa5ffe15a --- /dev/null +++ b/data/exp_b2/kv_compression_extended.json @@ -0,0 +1,189 @@ +{ + "config": { + "N_full": 2048, + "f_target": 0.9 + }, + "results": [ + { + "model": "EleutherAI/pythia-14m", + "tag": "pythia-14m", + "gamma": 0.685, + "phase": "A", + "D_f": 1466, + "N_full": 2048, + "f_target": 0.9, + "ppl_full": 78.36305186690407, + "ppl_at_df": 82.73959301570233, + "delta_ppl_df": 4.376541148798253, + "knee_heuristic": 1280, + "df_ratio": 0.7158203125, + "sweep": { + "64": 170.27565431091813, + "128": 129.87243953384964, + "256": 116.10089939684218, + "512": 100.39440165679781, + "768": 93.42492785289417, + "1024": 87.69748565849753, + "1280": 82.71468331445736, + "1466": 82.73959301570233, + "1536": 84.26238377642993, + "2048": 78.36305186690407 + } + }, + { + "model": "EleutherAI/pythia-70m", + "tag": "pythia-70m", + "gamma": 0.748, + "phase": "A", + "D_f": 1348, + "N_full": 2048, + "f_target": 0.9, + "ppl_full": 39.42330332269695, + "ppl_at_df": 38.29037532881002, + "delta_ppl_df": -1.1329279938869306, + "knee_heuristic": 1024, + "df_ratio": 0.658203125, + "sweep": { + "64": 78.26171993639424, + "128": 60.39432733839407, + "256": 53.77005947252492, + "512": 46.90251454572783, + "768": 43.56558823784614, + "1024": 41.0769018008491, + "1280": 39.57051656987743, + "1348": 38.29037532881002, + "1536": 41.25781274772638, + "2048": 39.42330332269695 + } + }, + { + "model": "EleutherAI/pythia-160m", + "tag": "pythia-160m", + "gamma": 0.511, + "phase": "A", + "D_f": 1651, + "N_full": 2048, + "f_target": 0.9, + "ppl_full": 22.49693114269694, + "ppl_at_df": 23.941233262843955, + "delta_ppl_df": 1.4443021201470145, + "knee_heuristic": 1024, + "df_ratio": 0.80615234375, + "sweep": { + "64": 45.05178022852514, + "128": 35.39264061338946, + "256": 30.868753730150203, + "512": 27.10712340123499, + "768": 25.202340040551977, + "1024": 23.906966979465913, + "1280": 22.180796411565147, + "1536": 23.290909891590083, + "1651": 23.941233262843955, + "2048": 22.49693114269694 + } + }, + { + "model": "EleutherAI/pythia-410m", + "tag": "pythia-410m", + "gamma": 1.022, + "phase": "B", + "D_f": 2048, + "N_full": 2048, + "f_target": 0.9, + "ppl_full": 13.986634758664158, + "ppl_at_df": 13.986634758664158, + "delta_ppl_df": 0.0, + "knee_heuristic": 768, + "df_ratio": 1.0, + "sweep": { + "64": 25.99515452550955, + "128": 20.497028981644508, + "256": 18.463651885474807, + "512": 15.943368404335239, + "768": 14.291077859966823, + "1024": 14.35273375572969, + "1280": 13.495131622333211, + "1536": 13.658299116074033, + "2048": 13.986634758664158 + } + }, + { + "model": "EleutherAI/pythia-1b", + "tag": "pythia-1b", + "gamma": 0.931, + "phase": "A", + "D_f": 445, + "N_full": 2048, + "f_target": 0.9, + "ppl_full": 11.797592766397049, + "ppl_at_df": 12.797310428095802, + "delta_ppl_df": 0.9997176616987531, + "knee_heuristic": 768, + "df_ratio": 0.21728515625, + "sweep": { + "64": 20.82336859802798, + "128": 16.34351822999527, + "256": 14.961815199990014, + "445": 12.797310428095802, + "512": 13.00719843117948, + "768": 11.944238442848128, + "1024": 11.90498469103213, + "1280": 11.308198053102004, + "1536": 11.367630159231787, + "2048": 11.797592766397049 + } + }, + { + "model": "EleutherAI/pythia-1.4b", + "tag": "pythia-1.4b", + "gamma": 0.705, + "phase": "A", + "D_f": 1433, + "N_full": 2048, + "f_target": 0.9, + "ppl_full": 10.531278903927081, + "ppl_at_df": 10.359741099193672, + "delta_ppl_df": -0.17153780473340952, + "knee_heuristic": 768, + "df_ratio": 0.69970703125, + "sweep": { + "64": 19.10579824421483, + "128": 14.338374193623926, + "256": 13.28459133971229, + "512": 11.393840662956944, + "768": 10.782337109011515, + "1024": 10.669041088499121, + "1280": 10.046359061813407, + "1433": 10.359741099193672, + "1536": 10.097998537270568, + "2048": 10.531278903927081 + } + }, + { + "model": "EleutherAI/pythia-2.8b", + "tag": "pythia-2.8b", + "gamma": 0.674, + "phase": "A", + "D_f": 1482, + "N_full": 2048, + "f_target": 0.9, + "ppl_full": 9.207173515380948, + "ppl_at_df": 8.86634974640192, + "delta_ppl_df": -0.34082376897902833, + "knee_heuristic": 768, + "df_ratio": 0.7236328125, + "sweep": { + "64": 16.845721029085166, + "128": 12.46358100808799, + "256": 11.516145416329916, + "512": 10.08633604056202, + "768": 9.462097751139718, + "1024": 9.502726676862784, + "1280": 8.677919928249965, + "1482": 8.86634974640192, + "1536": 8.778613874635514, + "2048": 9.207173515380948 + } + } + ] +} \ No newline at end of file diff --git a/data/exp_b2/kv_compression_results.json b/data/exp_b2/kv_compression_results.json new file mode 100644 index 0000000000000000000000000000000000000000..dee9f2b26680f998f025637fa49f791c1c40e6a7 --- /dev/null +++ b/data/exp_b2/kv_compression_results.json @@ -0,0 +1,190 @@ +{ + "config": { + "N_full": 2048, + "f_target": 0.9, + "eval_len": 64, + "n_samples": 120 + }, + "results": [ + { + "model": "EleutherAI/pythia-1b", + "tag": "pythia-1b", + "gamma": 0.931, + "phase": "A", + "N_full": 2048, + "D_f": 445, + "f_target": 0.9, + "ppl_full": 11.79557674849907, + "sweep": [ + { + "window": 64, + "ppl": 19.77381769411898, + "delta_ppl": 7.978240945619909, + "ratio": 0.03125 + }, + { + "window": 128, + "ppl": 16.397073349119978, + "delta_ppl": 4.601496600620909, + "ratio": 0.0625 + }, + { + "window": 256, + "ppl": 14.005542232108976, + "delta_ppl": 2.2099654836099063, + "ratio": 0.125 + }, + { + "window": 384, + "ppl": 12.949847142073574, + "delta_ppl": 1.154270393574505, + "ratio": 0.1875 + }, + { + "window": 445, + "ppl": 13.23990899615875, + "delta_ppl": 1.4443322476596805, + "ratio": 0.21728515625 + }, + { + "window": 512, + "ppl": 13.265685506123768, + "delta_ppl": 1.4701087576246987, + "ratio": 0.25 + }, + { + "window": 640, + "ppl": 12.585584646558791, + "delta_ppl": 0.7900078980597218, + "ratio": 0.3125 + }, + { + "window": 768, + "ppl": 12.062434404070897, + "delta_ppl": 0.26685765557182783, + "ratio": 0.375 + }, + { + "window": 1024, + "ppl": 11.65227339392187, + "delta_ppl": -0.14330335457719912, + "ratio": 0.5 + }, + { + "window": 1280, + "ppl": 11.561641727306759, + "delta_ppl": -0.2339350211923108, + "ratio": 0.625 + }, + { + "window": 1536, + "ppl": 11.190116400723584, + "delta_ppl": -0.6054603477754856, + "ratio": 0.75 + }, + { + "window": 1792, + "ppl": 11.750546375872558, + "delta_ppl": -0.04503037262651155, + "ratio": 0.875 + }, + { + "window": 2048, + "ppl": 11.79557674849907, + "delta_ppl": 0.0, + "ratio": 1.0 + } + ] + }, + { + "model": "EleutherAI/pythia-2.8b", + "tag": "pythia-2.8b", + "gamma": 0.674, + "phase": "A", + "N_full": 2048, + "D_f": 1482, + "f_target": 0.9, + "ppl_full": 9.242118727247712, + "sweep": [ + { + "window": 64, + "ppl": 15.474991362569368, + "delta_ppl": 6.232872635321655, + "ratio": 0.03125 + }, + { + "window": 128, + "ppl": 12.784194911370733, + "delta_ppl": 3.5420761841230206, + "ratio": 0.0625 + }, + { + "window": 256, + "ppl": 10.839832447967765, + "delta_ppl": 1.5977137207200531, + "ratio": 0.125 + }, + { + "window": 384, + "ppl": 10.115023661805676, + "delta_ppl": 0.8729049345579636, + "ratio": 0.1875 + }, + { + "window": 512, + "ppl": 10.232507490949049, + "delta_ppl": 0.9903887637013362, + "ratio": 0.25 + }, + { + "window": 640, + "ppl": 9.754356736249541, + "delta_ppl": 0.5122380090018286, + "ratio": 0.3125 + }, + { + "window": 768, + "ppl": 9.403105800965589, + "delta_ppl": 0.16098707371787668, + "ratio": 0.375 + }, + { + "window": 1024, + "ppl": 9.22952920678111, + "delta_ppl": -0.012589520466601556, + "ratio": 0.5 + }, + { + "window": 1280, + "ppl": 9.028686534058226, + "delta_ppl": -0.2134321931894867, + "ratio": 0.625 + }, + { + "window": 1482, + "ppl": 8.728037345559589, + "delta_ppl": -0.5140813816881238, + "ratio": 0.7236328125 + }, + { + "window": 1536, + "ppl": 8.6772931886815, + "delta_ppl": -0.5648255385662129, + "ratio": 0.75 + }, + { + "window": 1792, + "ppl": 9.120183357326662, + "delta_ppl": -0.1219353699210508, + "ratio": 0.875 + }, + { + "window": 2048, + "ppl": 9.242118727247712, + "delta_ppl": 0.0, + "ratio": 1.0 + } + ] + } + ] +} \ No newline at end of file diff --git a/data/exp_b3/band_ablation_results.json b/data/exp_b3/band_ablation_results.json new file mode 100644 index 0000000000000000000000000000000000000000..f7a441fc28e98e79fc5c9523aadaab3c678deb24 --- /dev/null +++ b/data/exp_b3/band_ablation_results.json @@ -0,0 +1,43 @@ +{ + "model": "EleutherAI/pythia-1b", + "k_dead": 23, + "rotary_ndims": 64, + "results": { + "baseline": { + "passkey": { + "50": 0.0, + "200": 0.0, + "500": 0.0 + }, + "lambada": { + "accuracy": 0.005, + "ppl": 454866.0959065663, + "n": 200 + } + }, + "zero_dead": { + "passkey": { + "50": 0.0, + "200": 0.0, + "500": 0.0 + }, + "lambada": { + "accuracy": 0.0, + "ppl": 283545.2363100814, + "n": 200 + } + }, + "zero_alive": { + "passkey": { + "50": 0.0, + "200": 0.0, + "500": 0.0 + }, + "lambada": { + "accuracy": 0.005, + "ppl": 2038171.0155264542, + "n": 200 + } + } + } +} \ No newline at end of file diff --git a/data/exp_b3/ppl_dissociation_results.json b/data/exp_b3/ppl_dissociation_results.json new file mode 100644 index 0000000000000000000000000000000000000000..444b90965baee0d06b92ffeab223f3c3b36554d8 --- /dev/null +++ b/data/exp_b3/ppl_dissociation_results.json @@ -0,0 +1,56 @@ +{ + "model": "EleutherAI/pythia-1b", + "k_dead": 23, + "windows": [ + 32, + 64, + 128, + 256, + 512, + 1024 + ], + "results": { + "baseline": { + "sweep": { + "32": 27.743402019949816, + "64": 21.773068760436132, + "128": 16.95406812662881, + "256": 16.020233852411856, + "512": 14.211691080630496, + "1024": 12.960885771516637 + }, + "ppl_semantic": 27.743402019949816, + "ppl_positional": 12.960885771516637, + "delta_32_1024": 14.78251624843318, + "delta_32_512": 13.53171093931932 + }, + "zero_dead": { + "sweep": { + "32": 377.73739000780085, + "64": 386.3479086430421, + "128": 382.87740392539627, + "256": 444.79663935459706, + "512": 512.9987648663821, + "1024": 551.7126470242462 + }, + "ppl_semantic": 377.73739000780085, + "ppl_positional": 551.7126470242462, + "delta_32_1024": -173.9752570164453, + "delta_32_512": -135.26137485858123 + }, + "zero_alive": { + "sweep": { + "32": 37.08881455526334, + "64": 29.76374421196143, + "128": 35.99531070851565, + "256": 108.26691457115533, + "512": 208.89791823047497, + "1024": 316.5837725060713 + }, + "ppl_semantic": 37.08881455526334, + "ppl_positional": 316.5837725060713, + "delta_32_1024": -279.49495795080793, + "delta_32_512": -171.80910367521165 + } + } +} \ No newline at end of file diff --git a/data/exp_c1/kcomp_results.json b/data/exp_c1/kcomp_results.json new file mode 100644 index 0000000000000000000000000000000000000000..cd96617f464717bea1f28635c4fbcecf9d2238d4 --- /dev/null +++ b/data/exp_c1/kcomp_results.json @@ -0,0 +1,239 @@ +[ + { + "model": "pythia-70m", + "N": 6, + "H": 8, + "L_crit": 4, + "K_profile": [ + 0.07763911867392617, + 0.07842551324142202, + 0.11314526562866627, + 0.20474838845550672, + 0.07090244267586222 + ], + "Q_profile": [ + 0.06610465848760559, + 0.09131361349339724, + 0.11400564907899831, + 0.227760425107776, + 0.06331452995704488 + ], + "V_profile": [ + 0.0616045756024556, + 0.08227625983737918, + 0.1146839495785792, + 0.21377603734256886, + 0.07926667136948652 + ], + "K_peak_layer": 4, + "Q_peak_layer": 4, + "V_peak_layer": 4, + "K_peak_offset": 0 + }, + { + "model": "gpt2-medium", + "N": 24, + "H": 16, + "L_crit": 23, + "K_profile": [ + 0.06103506451981622, + 0.10523412193748692, + 0.16475483350006614, + 0.11900680380161918, + 0.09023924498513756, + 0.09361903347793585, + 0.09214551916401335, + 0.0701573080243574, + 0.08136504761106099, + 0.08922410891061566, + 0.07441582263629326, + 0.09147854708327634, + 0.06895457141279705, + 0.0659302424369345, + 0.05667424262208149, + 0.05902057879468474, + 0.04604175178366454, + 0.04713842847110366, + 0.04637640086162432, + 0.04331456579426882, + 0.08827840651627214, + 0.045056660884065645, + 0.14481424126548015 + ], + "Q_profile": [ + 0.06580498527075962, + 0.13731173296375251, + 0.14237287053168904, + 0.10586680967779499, + 0.07628378444524508, + 0.07933409042096115, + 0.07765471506428213, + 0.07912749666259365, + 0.09888447934089316, + 0.08383711277278977, + 0.14898340701635443, + 0.08563871720737931, + 0.07044219739505277, + 0.07459395705703768, + 0.07015553693224641, + 0.07173814841906269, + 0.06648621005863456, + 0.08318085103256137, + 0.06337556813879544, + 0.05653237933403221, + 0.04426121828753696, + 0.06348301285973566, + 0.10714318593547484 + ], + "V_profile": [ + 0.07618968428659047, + 0.09952402213167591, + 0.12390326239793104, + 0.07044080905320751, + 0.09064447066344765, + 0.0840094669443566, + 0.07370686535362093, + 0.08881682206998061, + 0.07739126559080328, + 0.26710256339031285, + 0.17066172273155603, + 0.09194255026720133, + 0.10197375295447762, + 0.1016924889530299, + 0.08094143019334171, + 0.10554168952029899, + 0.06806749934192836, + 0.06967495768939082, + 0.05895280021070888, + 0.05935315851040146, + 0.055932962543690685, + 0.04316650334675105, + 0.05322525010769415 + ], + "K_peak_layer": 3, + "Q_peak_layer": 11, + "V_peak_layer": 10, + "K_peak_offset": -20 + }, + { + "model": "gpt2-large", + "N": 36, + "H": 20, + "L_crit": 35, + "K_profile": [ + 0.10204600655725315, + 0.10245304152063821, + 0.09945626982707968, + 0.1270507970137886, + 0.11461045234815166, + 0.08960367162733683, + 0.09230173972161326, + 0.10628865080433669, + 0.07615330513229007, + 0.05980954984677298, + 0.07467993487236117, + 0.06970073035921348, + 0.08472608985962225, + 0.059532571271955624, + 0.08578146303348434, + 0.09933364209601314, + 0.06928288499368919, + 0.05823642506382969, + 0.06366858669046595, + 0.05662700572111478, + 0.0517737386404463, + 0.06162472591229626, + 0.08923506546865921, + 0.05439551017277297, + 0.0450533548857805, + 0.03884783144557331, + 0.059079038036637305, + 0.04248835832050387, + 0.03916588203285821, + 0.03438998902102178, + 0.037364908124195816, + 0.0398013167135213, + 0.03430770113895285, + 0.031857288989281224, + 0.047316981801398754 + ], + "Q_profile": [ + 0.0926156463093199, + 0.12459792247930329, + 0.10905373935207564, + 0.13729902479073214, + 0.11073724797892695, + 0.11138277085052085, + 0.10269974796262528, + 0.10009156408479408, + 0.08981555550927472, + 0.06789127224796608, + 0.06635836772063745, + 0.0762821877550802, + 0.0842250910450107, + 0.06923863124505364, + 0.08615757928042567, + 0.07281506430037182, + 0.07910547275385217, + 0.07818330898061845, + 0.08213039034054928, + 0.07697603857549087, + 0.08219728977496428, + 0.08047540140225665, + 0.09346162207794718, + 0.06569473936981182, + 0.061446422519939654, + 0.056687870613025264, + 0.09368771470265813, + 0.07382922369377437, + 0.05525603873435406, + 0.04664965831524178, + 0.05443865802225085, + 0.06708424206876806, + 0.05864267486164044, + 0.05491950445151417, + 0.050896334920114035 + ], + "V_profile": [ + 0.14046555925966922, + 0.11324994239345679, + 0.12220646757838985, + 0.12557420640028613, + 0.0886604415833563, + 0.10846595330984804, + 0.11479623226706923, + 0.08065301990332517, + 0.07662240917805482, + 0.07928484292870548, + 0.09089848559551741, + 0.07899298302493246, + 0.08684749904333912, + 0.08980262842296552, + 0.09245004323942593, + 0.09257351921769463, + 0.09080265372193722, + 0.09678004098473368, + 0.08561795981812532, + 0.07997183056472142, + 0.1054498759145253, + 0.07706193459225787, + 0.11108227034804181, + 0.09103217013649004, + 0.060815293267904386, + 0.09049896756904828, + 0.13383090770105713, + 0.045097856175836525, + 0.042960990023855256, + 0.04926044853712369, + 0.04825473409369716, + 0.05995801013976544, + 0.05722076677874976, + 0.08760945466848434, + 0.09958350743484486 + ], + "K_peak_layer": 4, + "Q_peak_layer": 4, + "V_peak_layer": 1, + "K_peak_offset": -31 + } +] \ No newline at end of file diff --git a/data/exp_c2/induction_scores.json b/data/exp_c2/induction_scores.json new file mode 100644 index 0000000000000000000000000000000000000000..013821338787d6f124075c1e9d4074ffef1ad26e --- /dev/null +++ b/data/exp_c2/induction_scores.json @@ -0,0 +1,548 @@ +[ + { + "model": "pythia-70m", + "N": 6, + "H": 8, + "L_crit": 4, + "peak_induction_layer": 0, + "offset": -4, + "layer_max_scores": [ + 0.8958979249000549, + 0.024106914177536964, + 0.5235139727592468, + 0.024008391425013542, + 0.026467252522706985, + 0.006680313963443041 + ], + "scores_LH": [ + [ + 0.004235831554979086, + 9.187957766698673e-05, + 0.040488969534635544, + 0.8958979249000549, + 0.0008085652370937169, + 0.005655335728079081, + 0.008657287806272507, + 0.003135313279926777 + ], + [ + 0.014692787081003189, + 0.0022059439215809107, + 0.0056664710864424706, + 0.0002148010244127363, + 0.023661134764552116, + 0.003996202722191811, + 0.024106914177536964, + 0.008048777468502522 + ], + [ + 0.0033964505419135094, + 1.287177383346716e-06, + 0.002665276173502207, + 0.013739030808210373, + 0.0006762247066944838, + 0.029974132776260376, + 0.5235139727592468, + 7.5530069807427935e-06 + ], + [ + 0.013861700892448425, + 0.024008391425013542, + 0.00510761933401227, + 0.01669132150709629, + 0.0014215083792805672, + 0.00580417737364769, + 0.005511728581041098, + 0.008736595511436462 + ], + [ + 0.007067759521305561, + 0.002153001958504319, + 0.008509231731295586, + 0.026467252522706985, + 0.0005710781551897526, + 0.0004537889326456934, + 0.01982802338898182, + 0.013637621887028217 + ], + [ + 1.4357660802488681e-05, + 0.006680313963443041, + 0.00018700325745157897, + 0.0012544977944344282, + 0.00023468490689992905, + 9.029035297203336e-09, + 1.6711303032934666e-05, + 2.2903239305094303e-09 + ] + ] + }, + { + "model": "gpt2-medium", + "N": 24, + "H": 16, + "L_crit": 23, + "peak_induction_layer": 7, + "offset": -16, + "layer_max_scores": [ + 0.01947791688144207, + 0.6865736246109009, + 0.014291800558567047, + 0.013526566326618195, + 0.5871291160583496, + 0.00853385217487812, + 0.21638362109661102, + 0.888746976852417, + 0.07540667802095413, + 0.021465251222252846, + 0.03878360241651535, + 0.13735267519950867, + 0.03157110512256622, + 0.012717357836663723, + 0.03283996134996414, + 0.027218995615839958, + 0.018162474036216736, + 0.02137751877307892, + 0.015790745615959167, + 0.016747072339057922, + 0.009845052845776081, + 0.011436747387051582, + 0.01695154793560505, + 0.019688090309500694 + ], + "scores_LH": [ + [ + 0.013472379185259342, + 0.011570663191378117, + 0.01166445855051279, + 0.014548721723258495, + 0.011930054984986782, + 0.010027557611465454, + 0.009787365794181824, + 0.011641371063888073, + 0.01947791688144207, + 0.007912198081612587, + 0.009119994007050991, + 0.012400737963616848, + 0.011894762516021729, + 0.013383746147155762, + 0.012043224647641182, + 0.013485947623848915 + ], + [ + 0.010273352265357971, + 0.001711240503937006, + 0.02091180346906185, + 0.014163986779749393, + 0.6865736246109009, + 0.012486763298511505, + 0.013997252099215984, + 0.0007932193693704903, + 0.005385756026953459, + 0.011466982774436474, + 0.0084431953728199, + 0.020845884457230568, + 0.022297821938991547, + 0.004162461031228304, + 0.0028727841563522816, + 0.00865744799375534 + ], + [ + 0.008288095705211163, + 0.014291800558567047, + 0.0007500517531298101, + 9.979950846172869e-05, + 3.1433632102562115e-05, + 0.00024235770979430526, + 8.54063400765881e-05, + 0.0015347973676398396, + 0.00098496547434479, + 0.0006653810851275921, + 5.4627071222057566e-05, + 0.011361389420926571, + 0.0014283874770626426, + 0.0038152504712343216, + 0.0015850246418267488, + 0.00014867258141748607 + ], + [ + 0.013526566326618195, + 2.858385232684668e-05, + 1.0109235972777242e-06, + 5.359258370418729e-09, + 0.00015067598724272102, + 1.6373614926123992e-06, + 0.007509463932365179, + 6.417651457013562e-05, + 6.01602252459088e-08, + 4.670388989325147e-06, + 1.1010952505330351e-07, + 2.4040537027758546e-06, + 0.013181107118725777, + 6.787542750075204e-12, + 0.0009738670778460801, + 7.652112543610201e-08 + ], + [ + 0.010118700563907623, + 4.330320734879933e-06, + 3.3766966112125374e-07, + 5.139944292409382e-08, + 6.737444664395298e-07, + 3.995830411440693e-06, + 0.5871291160583496, + 6.447474788728869e-06, + 0.00011301909398753196, + 0.0035340441390872, + 0.010865827091038227, + 8.741832857595e-07, + 6.859724521746102e-07, + 6.425371280172582e-22, + 1.2227254046592861e-05, + 0.003389840479940176 + ], + [ + 0.0011563701555132866, + 0.00014799382188357413, + 1.3975404726807028e-05, + 9.359767432215449e-07, + 1.951531430677278e-06, + 0.00853385217487812, + 9.335150252809399e-07, + 0.00044699289719574153, + 0.0012824477162212133, + 4.150431777816266e-05, + 0.0026782613713294268, + 1.4544803092720067e-17, + 0.006188175641000271, + 0.0004336057754699141, + 0.004623007960617542, + 7.239461865538033e-07 + ], + [ + 0.018096070736646652, + 0.0013204844435676932, + 0.007576430216431618, + 0.004701701458543539, + 0.21638362109661102, + 0.004617438651621342, + 2.0043528365931707e-06, + 3.0039235099366124e-08, + 6.231999577721581e-05, + 0.0016532540321350098, + 5.3159452363615856e-05, + 0.007146933116018772, + 0.005562087055295706, + 0.0033350090961903334, + 0.00026481968234293163, + 4.253764927852899e-06 + ], + [ + 0.03647330030798912, + 5.3868066061113495e-06, + 0.0002692818234208971, + 0.006354471668601036, + 0.002618814818561077, + 0.017090119421482086, + 8.830003025650512e-06, + 0.0032796552404761314, + 0.019297800958156586, + 0.0009823645232245326, + 7.754172111162916e-05, + 0.888746976852417, + 0.0009748293086886406, + 4.0786831959849223e-05, + 0.0007570453453809023, + 0.009391972795128822 + ], + [ + 1.206413799081929e-05, + 5.389630359786679e-07, + 4.046598041895777e-05, + 0.0005342984804883599, + 0.005691627971827984, + 0.01540854386985302, + 9.281576240027789e-06, + 9.822638276091311e-06, + 0.0017763616051524878, + 0.005733284633606672, + 0.017342235893011093, + 0.0037897853180766106, + 0.00010528390703257173, + 0.01485004834830761, + 0.051943451166152954, + 0.07540667802095413 + ], + [ + 0.0012536238646134734, + 0.00813805591315031, + 0.00014916836516931653, + 0.008670263923704624, + 0.021465251222252846, + 9.668528946349397e-05, + 0.017977319657802582, + 0.015283404849469662, + 0.0006169080152176321, + 0.005561241880059242, + 7.948989514261484e-05, + 0.0004986606072634459, + 0.0016663658898323774, + 0.00935372058302164, + 0.0007955027977004647, + 2.775281245703809e-05 + ], + [ + 0.01642271690070629, + 0.00520413788035512, + 0.008865851908922195, + 0.005196448881179094, + 0.010900100693106651, + 0.0012729710433632135, + 0.008778614923357964, + 0.00021764379926025867, + 0.03878360241651535, + 0.001199488528072834, + 0.000294915575068444, + 0.010118518956005573, + 0.02512076497077942, + 0.0056185368448495865, + 0.023154281079769135, + 0.0013470664853230119 + ], + [ + 0.006713863927870989, + 0.01216935645788908, + 6.527941150125116e-05, + 0.019739700481295586, + 0.004458228591829538, + 0.005823900923132896, + 9.272093302570283e-05, + 0.001509197405539453, + 0.009352856315672398, + 0.01335617620497942, + 0.011705056764185429, + 0.0036970737855881453, + 0.0016537178307771683, + 0.004556253086775541, + 0.13735267519950867, + 0.006310749799013138 + ], + [ + 0.013026967644691467, + 0.019136730581521988, + 0.0012070407392457128, + 0.011290468275547028, + 0.010486502200365067, + 0.004635741002857685, + 0.0026256018318235874, + 0.005234092008322477, + 0.0037560737691819668, + 0.004262288101017475, + 0.0003435182443354279, + 0.002484071534126997, + 0.015279185958206654, + 0.03157110512256622, + 0.010532112792134285, + 0.0008488123421557248 + ], + [ + 0.011602484621107578, + 0.0027499415446072817, + 0.0005833106115460396, + 0.0006004857132211328, + 0.0003504140768200159, + 5.752416473114863e-05, + 0.0036196312867105007, + 0.00039095268584787846, + 0.0031686422880738974, + 0.007973864674568176, + 0.00025579670909792185, + 0.009425978176295757, + 1.4617036868003197e-05, + 0.00019438040908426046, + 0.012717357836663723, + 0.0035256037954241037 + ], + [ + 0.007017839699983597, + 5.054979556007311e-05, + 0.02335897460579872, + 0.00013514926831703633, + 0.00017183057207148522, + 0.0020430292934179306, + 0.004447019658982754, + 0.009814447723329067, + 0.0009680199436843395, + 0.03283996134996414, + 0.006121987476944923, + 0.008823960088193417, + 8.22612491901964e-05, + 0.00512811541557312, + 0.00016890274127945304, + 0.0038288908544927835 + ], + [ + 0.007945925928652287, + 0.004497022833675146, + 0.0036060449201613665, + 0.004188410472124815, + 0.0059702773578464985, + 0.005060691386461258, + 0.00436297757551074, + 0.027218995615839958, + 0.00016586788115091622, + 0.002148669445887208, + 0.00024048970954027027, + 0.004689166788011789, + 0.01785413734614849, + 0.0028160102665424347, + 0.00828561931848526, + 7.01921817380935e-05 + ], + [ + 0.00023207688354887068, + 0.004151418339461088, + 0.004216373898088932, + 0.0021413222420960665, + 4.725692269857973e-05, + 6.44422834739089e-05, + 0.0012517935829237103, + 0.00029063067631796, + 0.005691335536539555, + 0.0028251269832253456, + 0.0034938573371618986, + 0.002755748573690653, + 0.018162474036216736, + 0.01187220774590969, + 0.0029147102031856775, + 0.007940863259136677 + ], + [ + 0.008305597119033337, + 0.012066308408975601, + 0.016016943380236626, + 0.0050119804218411446, + 0.00035185503656975925, + 0.00040175471804104745, + 0.003968107048422098, + 0.011314893141388893, + 0.005298938136547804, + 0.0004368901427369565, + 0.0009932030225172639, + 8.521933341398835e-05, + 0.0004254616505932063, + 5.175965998205356e-05, + 0.02137751877307892, + 0.003152623074129224 + ], + [ + 0.015790745615959167, + 0.011077925562858582, + 0.00014835598994977772, + 0.00039545708568766713, + 0.004170551896095276, + 0.0008766044629737735, + 0.00012335111387073994, + 0.009392562322318554, + 0.006709921173751354, + 0.000358004734152928, + 0.0037581482902169228, + 0.009491664357483387, + 0.008369429968297482, + 0.004449060186743736, + 0.004326361697167158, + 0.00463701831176877 + ], + [ + 0.007132203783839941, + 0.000447290251031518, + 0.003016797360032797, + 0.0029673376120626926, + 0.006409043446183205, + 0.0061729843728244305, + 0.016747072339057922, + 0.0022946905810385942, + 0.007488078437745571, + 0.003381514921784401, + 0.011582648381590843, + 0.0022400980815291405, + 0.005024586338549852, + 0.00275859865359962, + 0.006567580159753561, + 0.004421334248036146 + ], + [ + 0.006971921771764755, + 0.00841355137526989, + 0.003139962675049901, + 0.003939655609428883, + 0.0016310332575812936, + 0.004540630150586367, + 0.003662562696263194, + 0.0020230025984346867, + 0.006552675738930702, + 0.002649497240781784, + 0.009845052845776081, + 0.007338748779147863, + 0.0016322721494361758, + 0.00917451735585928, + 0.004284942522644997, + 0.005528532899916172 + ], + [ + 0.005867542698979378, + 0.010249069891870022, + 0.004494961351156235, + 0.0013907018583267927, + 0.00683134188875556, + 0.005067203659564257, + 0.004023923072963953, + 0.011436747387051582, + 0.0017360950587317348, + 0.004243298899382353, + 0.007778732106089592, + 0.003580218879505992, + 0.004791188519448042, + 0.008327461779117584, + 0.005117479711771011, + 0.007851925678551197 + ], + [ + 0.004957208409905434, + 0.006175430491566658, + 0.011691322550177574, + 0.012195643037557602, + 0.012567561119794846, + 0.00579296937212348, + 0.008865961804986, + 0.004032428376376629, + 0.012407738715410233, + 0.004165831953287125, + 0.00979010108858347, + 0.006625777576118708, + 0.011179488152265549, + 0.009101040661334991, + 0.0007258547702804208, + 0.01695154793560505 + ], + [ + 0.013907046988606453, + 0.009370146319270134, + 0.005360445939004421, + 0.004168189130723476, + 0.003616396803408861, + 0.006632850971072912, + 0.019688090309500694, + 0.009024055674672127, + 0.012836091220378876, + 0.011910864152014256, + 0.008653552271425724, + 0.005626976024359465, + 0.010370035655796528, + 0.00567439803853631, + 0.012067004106938839, + 0.014186697080731392 + ] + ] + } +] \ No newline at end of file diff --git a/data/exp_corpus_gamma_local/results.json b/data/exp_corpus_gamma_local/results.json new file mode 100644 index 0000000000000000000000000000000000000000..63e690be3f9374d6fc48e1f3bf5b86becd02c059 --- /dev/null +++ b/data/exp_corpus_gamma_local/results.json @@ -0,0 +1,670 @@ +{ + "base_model": "EleutherAI/pythia-1b", + "ft_steps": 300, + "lr": 5e-06, + "lora_r": 8, + "runs": [ + { + "corpus": "code", + "ft_info": { + "final_loss": 1.5786903488636017, + "loss_curve": [ + 1.7335025072097778, + 1.9803385734558105, + 1.8001309633255005, + 1.3697925806045532, + 1.8405475616455078, + 1.8691179752349854, + 1.9074724912643433, + 1.0950721502304077, + 1.3699240684509277, + 1.8821594715118408, + 2.024385452270508, + 1.369756817817688, + 1.5220543146133423, + 1.5606002807617188, + 1.1731555461883545, + 1.7953110933303833, + 1.1576684713363647, + 1.5628548860549927, + 1.79978609085083, + 1.1730200052261353, + 1.8434703350067139, + 1.906765341758728, + 1.9397817850112915, + 2.116682291030884, + 1.7992839813232422, + 1.783668041229248, + 1.7211863994598389, + 1.5991277694702148, + 1.8962825536727905, + 1.886739730834961, + 1.5597577095031738, + 1.3437697887420654, + 1.9061907529830933, + 1.385431170463562, + 1.794171929359436, + 1.8398511409759521, + 1.436535358428955, + 1.4555039405822754, + 1.7916845083236694, + 1.4449630975723267, + 1.4364075660705566, + 1.2867382764816284, + 1.8858691453933716, + 1.1567102670669556, + 1.3646687269210815, + 1.8392001390457153, + 1.166734218597412, + 1.8420894145965576, + 1.4443013668060303, + 1.3643676042556763, + 1.8804774284362793, + 1.1719647645950317, + 1.6620793342590332, + 1.8799571990966797, + 1.6121231317520142, + 1.4746389389038086, + 1.6120548248291016, + 2.114345073699951, + 2.1678085327148438, + 1.4356815814971924, + 1.5612274408340454, + 1.3819233179092407, + 1.084471344947815, + 1.5582382678985596, + 1.9779322147369385, + 1.6614772081375122, + 1.1713000535964966, + 1.7821418046951294, + 2.2096822261810303, + 1.6235990524291992, + 1.7318174839019775, + 1.7059086561203003, + 1.6718889474868774, + 1.4544485807418823, + 1.8111674785614014, + 1.6879441738128662, + 1.7187929153442383, + 1.5970934629440308, + 1.4740554094314575, + 1.560376763343811, + 1.5200996398925781, + 1.9037580490112305, + 1.2383205890655518, + 2.2088112831115723, + 1.170607328414917, + 1.4736483097076416, + 1.9034537076950073, + 1.4536621570587158, + 1.8782600164413452, + 1.6605390310287476, + 1.8369134664535522, + 1.0837067365646362, + 1.7307415008544922, + 1.622525691986084, + 1.8105442523956299, + 1.2855429649353027, + 1.8101798295974731, + 1.6101410388946533, + 1.8399282693862915, + 1.5191009044647217, + 1.8363447189331055, + 1.8841501474380493, + 2.02142333984375, + 1.892640233039856, + 1.4732469320297241, + 2.1118950843811035, + 1.7300200462341309, + 1.2850699424743652, + 1.5565695762634277, + 1.3806089162826538, + 1.8665953874588013, + 1.2849946022033691, + 1.0831176042556763, + 1.6595441102981567, + 1.8353195190429688, + 1.7291887998580933, + 1.8830633163452148, + 1.7804332971572876, + 1.9013572931289673, + 2.0197598934173584, + 2.237935781478882, + 1.6216524839401245, + 1.2372723817825317, + 1.2370171546936035, + 1.65549898147583, + 1.9009029865264893, + 1.759272575378418, + 1.5586951971054077, + 1.4341216087341309, + 2.236729621887207, + 1.5316920280456543, + 1.8343524932861328, + 1.4723247289657593, + 1.838438630104065, + 1.6859309673309326, + 1.6692949533462524, + 1.5947413444519043, + 2.2358243465423584, + 1.8378887176513672, + 1.8894621133804321, + 1.7525148391723633, + 1.5308793783187866, + 1.668938159942627, + 1.8890196084976196, + 1.0820047855377197, + 1.1647138595581055, + 2.1648905277252197, + 1.0936449766159058, + 1.864848256111145, + 1.365576148033142, + 1.7790507078170776, + 1.5547784566879272, + 2.164261817932129, + 1.365126371383667, + 1.7893743515014648, + 1.1534713506698608, + 1.5186868906021118, + 1.7890501022338867, + 1.5297833681106567, + 1.8079732656478882, + 1.8989531993865967, + 1.093328833580017, + 1.9366437196731567, + 1.7519009113311768, + 1.1691677570343018, + 1.4420546293258667, + 1.6543174982070923, + 1.4518083333969116, + 1.9743648767471313, + 1.1636959314346313, + 1.726857304573059, + 1.2835420370101929, + 2.003051996231079, + 2.0175607204437256, + 1.1527361869812012, + 2.108022928237915, + 1.4712754487991333, + 1.6583722829818726, + 1.6583576202392578, + 2.1076576709747314, + 1.080839991569519, + 0.6421031951904297, + 2.1628992557525635, + 1.7779661417007446, + 1.4512592554092407, + 1.7886227369308472, + 2.0169899463653564, + 1.3793656826019287, + 1.973334789276123, + 2.01643967628479, + 1.666930079460144, + 1.5533527135849, + 1.4413295984268188, + 1.832270860671997, + 1.2352150678634644, + 1.8864606618881226, + 1.3605445623397827, + 1.9730943441390991, + 2.206134080886841, + 1.1519991159439087, + 1.2829445600509644, + 1.0802873373031616, + 1.7882617712020874, + 1.3789080381393433, + 1.6664576530456543, + 1.3788015842437744, + 2.16204571723938, + 1.9358717203140259, + 1.440925121307373, + 1.3165771961212158, + 1.9357376098632812, + 1.8748422861099243, + 1.4507070779800415, + 1.7253570556640625, + 2.0217010974884033, + 1.6198580265045166, + 1.7253906726837158, + 1.9355254173278809, + 2.1056647300720215, + 1.7145726680755615, + 1.1628446578979492, + 1.806989312171936, + 1.4405674934387207, + 1.2826590538024902, + 1.5525647401809692, + 1.2346805334091187, + 0.6419479250907898, + 1.7876511812210083, + 2.1054205894470215, + 2.2321364879608154, + 1.750936508178711, + 1.7921675443649292, + 1.9717512130737305, + 1.8067201375961304, + 1.6657570600509644, + 1.6194348335266113, + 1.4330564737319946, + 2.0211901664733887, + 1.8627406358718872, + 1.6531606912612915, + 1.862587332725525, + 1.665451169013977, + 1.6842645406723022, + 1.1682631969451904, + 1.1684114933013916, + 1.378068447113037, + 1.7506420612335205, + 1.7566165924072266, + 1.4705617427825928, + 1.4401676654815674, + 1.5520334243774414, + 1.2344229221343994, + 1.0922908782958984, + 1.0921638011932373, + 1.5144391059875488, + 1.208531141281128, + 1.7768268585205078, + 1.2822725772857666, + 2.1613523960113525, + 0.6418479084968567, + 1.4501129388809204, + 2.1046218872070312, + 1.1511739492416382, + 1.8350486755371094, + 1.551810622215271, + 1.4399678707122803, + 1.3777945041656494, + 1.8811619281768799, + 1.7138186693191528, + 1.5923242568969727, + 2.020629405975342, + 1.5278847217559814, + 1.5143862962722778, + 1.8851925134658813, + 1.8062182664871216, + 1.3793983459472656, + 1.5172584056854248, + 0.6415963172912598, + 1.379799723625183, + 1.971291422843933, + 1.8620975017547607, + 1.89655339717865, + 1.4698920249938965, + 1.3592877388000488, + 1.7914248704910278, + 1.7863967418670654, + 1.7763843536376953, + 1.8964570760726929, + 1.7862087488174438, + 1.8966684341430664, + 1.786729097366333, + 1.4699108600616455, + 1.5922114849090576, + 1.7245607376098633, + 1.6525399684906006, + 1.8966765403747559, + 1.4699344635009766, + 1.5170170068740845, + 1.5560373067855835, + 1.4498573541641235 + ] + }, + "post_gamma": { + "gamma": 0.966178741392759, + "r2": 0.996691590496258, + "means": { + "10": 0.008440410075709224, + "20": 0.006177946208044887, + "30": 0.004625266371294856, + "50": 0.0025753298308700324, + "100": 0.0014661711361259222, + "200": 0.0006166024971753359, + "500": 0.0003020010609179735, + "1000": 0.00015268372371792793 + } + }, + "delta_vs_baseline": -0.0001944911767313151 + }, + { + "corpus": "narrative", + "ft_info": { + "final_loss": 1.871248276233673, + "loss_curve": [ + 1.8850247859954834, + 1.5679991245269775, + 1.699575662612915, + 1.929004430770874, + 1.8659214973449707, + 2.046631336212158, + 1.868190050125122, + 1.8008846044540405, + 1.9290558099746704, + 1.7464295625686646, + 2.005218029022217, + 1.9288133382797241, + 2.056614398956299, + 1.654632329940796, + 1.939730167388916, + 1.9408953189849854, + 1.5707067251205444, + 1.6992216110229492, + 1.6541463136672974, + 1.8722559213638306, + 1.8671048879623413, + 1.904943585395813, + 1.9479142427444458, + 1.6983059644699097, + 1.8683969974517822, + 2.011848211288452, + 2.0581777095794678, + 1.9030776023864746, + 1.840545415878296, + 2.0542476177215576, + 2.0537948608398438, + 1.8650625944137573, + 1.9916784763336182, + 1.9368799924850464, + 1.8623170852661133, + 1.7201260328292847, + 1.876646876335144, + 1.8392106294631958, + 1.9373348951339722, + 2.0459842681884766, + 1.8608137369155884, + 1.8965306282043457, + 1.8688678741455078, + 2.0450353622436523, + 1.742274522781372, + 1.6500853300094604, + 1.843599557876587, + 1.7418166399002075, + 1.9123096466064453, + 1.9121408462524414, + 1.9420539140701294, + 2.0339572429656982, + 1.5650535821914673, + 1.729468822479248, + 1.899993896484375, + 2.049574375152588, + 1.562070608139038, + 1.841815710067749, + 1.6479467153549194, + 1.8630106449127197, + 1.8640869855880737, + 1.8775439262390137, + 1.845837950706482, + 1.948379635810852, + 1.8222053050994873, + 2.005174398422241, + 2.0520715713500977, + 1.767685055732727, + 1.5625134706497192, + 2.0194313526153564, + 1.85781729221344, + 1.8623008728027344, + 1.6451416015625, + 1.8573181629180908, + 1.7376720905303955, + 1.8387165069580078, + 1.8539414405822754, + 1.8954395055770874, + 1.8741995096206665, + 1.9454344511032104, + 1.8696774244308472, + 1.945141077041626, + 1.9061145782470703, + 1.86259925365448, + 2.0171022415161133, + 1.8520841598510742, + 1.8324759006500244, + 1.9945802688598633, + 1.8940138816833496, + 1.763547420501709, + 1.933668613433838, + 1.871358871459961, + 1.8672291040420532, + 2.0426530838012695, + 1.7228113412857056, + 2.035040855407715, + 1.8663686513900757, + 1.8921154737472534, + 1.834435224533081, + 1.8488351106643677, + 1.869310975074768, + 1.8301194906234741, + 1.8552448749542236, + 1.8510816097259521, + 1.9916532039642334, + 1.750362753868103, + 1.667129635810852, + 1.8502215147018433, + 1.5561504364013672, + 1.7494525909423828, + 1.9102181196212769, + 1.8459244966506958, + 1.7591511011123657, + 1.8581483364105225, + 1.814934492111206, + 2.0438878536224365, + 1.7483927011489868, + 1.8569954633712769, + 1.8889272212982178, + 1.8723409175872803, + 1.9087202548980713, + 1.8885544538497925, + 1.8871246576309204, + 1.886452078819275, + 2.02205753326416, + 1.7842527627944946, + 2.0298373699188232, + 1.9103327989578247, + 1.850628137588501, + 2.036386489868164, + 2.0208280086517334, + 1.9099643230438232, + 1.9203553199768066, + 1.9224164485931396, + 1.9199655055999756, + 1.9061506986618042, + 1.93528151512146, + 1.8441433906555176, + 1.7819486856460571, + 1.8886950016021729, + 1.8688747882843018, + 1.6356078386306763, + 1.6617672443389893, + 1.548721194267273, + 1.8836687803268433, + 1.8611105680465698, + 1.8586962223052979, + 1.9853556156158447, + 1.9198824167251587, + 1.9229081869125366, + 1.753879427909851, + 1.8261042833328247, + 1.8258929252624512, + 1.9224885702133179, + 1.8822396993637085, + 1.8786325454711914, + 2.0165605545043945, + 1.846158742904663, + 1.7025809288024902, + 1.983391284942627, + 1.713325023651123, + 1.5460034608840942, + 1.9830138683319092, + 2.03104567527771, + 1.836449146270752, + 1.8817373514175415, + 2.0226802825927734, + 1.545167088508606, + 1.851143717765808, + 1.9170336723327637, + 1.8548890352249146, + 1.879260778427124, + 1.700899600982666, + 1.711281657218933, + 1.7108044624328613, + 2.013983964920044, + 1.8828235864639282, + 1.8962332010269165, + 1.8824502229690552, + 1.725631594657898, + 1.8565212488174438, + 2.0056545734405518, + 1.8557496070861816, + 1.8819098472595215, + 1.9167938232421875, + 1.9880963563919067, + 1.8795326948165894, + 1.9292597770690918, + 1.8523396253585815, + 2.028334140777588, + 1.8737818002700806, + 1.698586106300354, + 1.915676236152649, + 1.7396495342254639, + 1.8617658615112305, + 1.6766961812973022, + 1.5412507057189941, + 1.9278273582458496, + 2.00399112701416, + 2.0209150314331055, + 1.6554280519485474, + 2.020328998565674, + 1.8049713373184204, + 1.6292237043380737, + 1.6292558908462524, + 1.7078008651733398, + 1.8607878684997559, + 1.747053861618042, + 2.025662899017334, + 1.7736656665802002, + 1.7737804651260376, + 2.002042531967163, + 1.952539086341858, + 1.8399826288223267, + 1.8499609231948853, + 2.0095813274383545, + 1.8713734149932861, + 1.9127546548843384, + 1.9124735593795776, + 1.8463691473007202, + 2.0244252681732178, + 1.7060317993164062, + 1.8183873891830444, + 1.984102487564087, + 2.031738758087158, + 2.0016281604766846, + 1.8969404697418213, + 2.0006048679351807, + 1.8762173652648926, + 1.9252687692642212, + 1.9697965383529663, + 1.8699369430541992, + 1.9690475463867188, + 1.537975549697876, + 2.017864942550659, + 1.833168387413025, + 1.7445636987686157, + 2.015439748764038, + 1.6736130714416504, + 1.9076766967773438, + 1.8376405239105225, + 1.8324910402297974, + 1.9072725772857666, + 1.8330106735229492, + 1.6948304176330566, + 1.7443766593933105, + 2.030383825302124, + 1.8495045900344849, + 1.6518797874450684, + 1.8320437669754028, + 1.7435811758041382, + 1.5412589311599731, + 2.0147323608398438, + 1.7433724403381348, + 1.8766933679580688, + 2.0163280963897705, + 1.827210783958435, + 1.9990049600601196, + 1.8570488691329956, + 1.843629240989685, + 2.021831750869751, + 1.770825743675232, + 1.7037043571472168, + 1.8014593124389648, + 1.8018332719802856, + 1.949277639389038, + 2.0160651206970215, + 1.9493186473846436, + 2.0288262367248535, + 1.9819650650024414, + 1.7203139066696167, + 1.7203222513198853, + 1.9983279705047607, + 1.9089336395263672, + 1.8882232904434204, + 1.909438133239746, + 2.028724193572998, + 1.8716741800308228, + 1.7031538486480713, + 1.6507591009140015, + 1.9743309020996094, + 1.873874306678772, + 1.8739709854125977, + 1.7198046445846558, + 1.6508300304412842, + 1.8950600624084473, + 1.703025221824646, + 2.013514757156372, + 1.7199316024780273, + 2.0155608654022217, + 1.9816991090774536, + 1.9818196296691895, + 2.0220911502838135, + 1.9670374393463135, + 1.769582986831665, + 1.9090226888656616, + 1.8901435136795044, + 1.8265856504440308, + 1.87111496925354, + 2.015575408935547 + ] + }, + "post_gamma": { + "gamma": 0.977477133040617, + "r2": 0.9967344078174232, + "means": { + "10": 0.008457934316247702, + "20": 0.0061964262835681435, + "30": 0.004644344476982951, + "50": 0.002566973390057683, + "100": 0.0014542452432215213, + "200": 0.0006053818855434656, + "500": 0.00029141098260879514, + "1000": 0.0001480217557400465 + } + }, + "delta_vs_baseline": 0.011103900471126638 + } + ], + "baseline": { + "gamma": 0.9663732325694904, + "r2": 0.9966110527283811, + "means": { + "10": 0.008427766161039471, + "20": 0.006169456802308559, + "30": 0.004612918393686414, + "50": 0.002562444470822811, + "100": 0.0014617788139730692, + "200": 0.000613459562882781, + "500": 0.0003011568635702133, + "1000": 0.00015195895917713642 + } + }, + "verdict": "Z REFUTED \u2014 gamma is arch-locked (|delta_AB| <= 0.02)", + "delta_AB": 0.011298391647857953 +} \ No newline at end of file diff --git a/data/exp_d1/grokking_hagedorn_results.json b/data/exp_d1/grokking_hagedorn_results.json new file mode 100644 index 0000000000000000000000000000000000000000..944d77ed41cc2b7161b17adb38d92ca7e2475415 --- /dev/null +++ b/data/exp_d1/grokking_hagedorn_results.json @@ -0,0 +1,1214 @@ +{ + "config": { + "P": 113, + "d_model": 128, + "n_heads": 4, + "n_layers": 2, + "frac_train": 0.3, + "n_steps": 15000, + "lr": 0.001, + "wd": 1.0 + }, + "history": [ + { + "step": 100, + "train_acc": 0.2010443864229765, + "val_acc": 0.001006824029533505, + "gamma": 0.4565135052603194, + "C_V": 1.8399721238316704, + "loss": 3.825352668762207 + }, + { + "step": 200, + "train_acc": 0.9266318537859007, + "val_acc": 0.023156952679270613, + "gamma": 0.3895136856668784, + "C_V": 1.6380383581446414, + "loss": 1.504637360572815 + }, + { + "step": 300, + "train_acc": 1.0, + "val_acc": 0.07786105828392438, + "gamma": 0.3496480570612192, + "C_V": 1.537628988207901, + "loss": 0.14490902423858643 + }, + { + "step": 400, + "train_acc": 1.0, + "val_acc": 0.10672334713055151, + "gamma": 0.38321624094820267, + "C_V": 1.6213137673036886, + "loss": 0.10841263085603714 + }, + { + "step": 500, + "train_acc": 1.0, + "val_acc": 0.12395122496923593, + "gamma": 0.3950266613573821, + "C_V": 1.6529654054568845, + "loss": 0.08308350294828415 + }, + { + "step": 600, + "train_acc": 1.0, + "val_acc": 0.12797852108736996, + "gamma": 0.3333914616877184, + "C_V": 1.5001308002021672, + "loss": 0.06850433349609375 + }, + { + "step": 700, + "train_acc": 1.0, + "val_acc": 0.14956930305403288, + "gamma": 0.3652403531715452, + "C_V": 1.5753994523698072, + "loss": 0.06506888568401337 + }, + { + "step": 800, + "train_acc": 0.8360313315926893, + "val_acc": 0.042062870567177536, + "gamma": 0.39133315334432345, + "C_V": 1.6429348920423472, + "loss": 1.5839844942092896 + }, + { + "step": 900, + "train_acc": 1.0, + "val_acc": 0.16355297013088713, + "gamma": 0.3964798772470647, + "C_V": 1.6569455802708548, + "loss": 0.056978169828653336 + }, + { + "step": 1000, + "train_acc": 1.0, + "val_acc": 0.1826826266920237, + "gamma": 0.37794320125001785, + "C_V": 1.6075702444045168, + "loss": 0.07091519236564636 + }, + { + "step": 1100, + "train_acc": 1.0, + "val_acc": 0.16254614610135362, + "gamma": 0.4241111726620453, + "C_V": 1.7364462592936531, + "loss": 0.08583323657512665 + }, + { + "step": 1200, + "train_acc": 1.0, + "val_acc": 0.1945407763731961, + "gamma": 0.4069948379131102, + "C_V": 1.6863259612797021, + "loss": 0.06576251238584518 + }, + { + "step": 1300, + "train_acc": 1.0, + "val_acc": 0.21020248349927284, + "gamma": 0.39445129961170516, + "C_V": 1.6513948413377353, + "loss": 0.06940054893493652 + }, + { + "step": 1400, + "train_acc": 1.0, + "val_acc": 0.18693366148338741, + "gamma": 0.4057934302503516, + "C_V": 1.6829164316061347, + "loss": 0.09401404857635498 + }, + { + "step": 1500, + "train_acc": 1.0, + "val_acc": 0.22183689450721558, + "gamma": 0.40753187922344875, + "C_V": 1.6878545274120311, + "loss": 0.05496717989444733 + }, + { + "step": 1600, + "train_acc": 1.0, + "val_acc": 0.2321288734757803, + "gamma": 0.39177189833145865, + "C_V": 1.6441200221704946, + "loss": 0.08135832101106644 + }, + { + "step": 1700, + "train_acc": 0.2422976501305483, + "val_acc": 0.013983667076854235, + "gamma": 0.4234029498881147, + "C_V": 1.7343134166329082, + "loss": 3.3658838272094727 + }, + { + "step": 1800, + "train_acc": 1.0, + "val_acc": 0.2133348249244882, + "gamma": 0.4110662114817644, + "C_V": 1.6979837453646733, + "loss": 0.13020755350589752 + }, + { + "step": 1900, + "train_acc": 1.0, + "val_acc": 0.2515941380467614, + "gamma": 0.4014263559643965, + "C_V": 1.6706382079537725, + "loss": 0.05627824738621712 + }, + { + "step": 2000, + "train_acc": 1.0, + "val_acc": 0.25808255957042175, + "gamma": 0.39498778391778283, + "C_V": 1.6528591876632566, + "loss": 0.0826713889837265 + }, + { + "step": 2100, + "train_acc": 1.0, + "val_acc": 0.25439087146213224, + "gamma": 0.38013394134492434, + "C_V": 1.6132517437229932, + "loss": 0.10779036581516266 + }, + { + "step": 2200, + "train_acc": 0.9814621409921671, + "val_acc": 0.18615057612708358, + "gamma": 0.37211057774414213, + "C_V": 1.5926371181843406, + "loss": 0.7241863012313843 + }, + { + "step": 2300, + "train_acc": 1.0, + "val_acc": 0.29611813401946524, + "gamma": 0.3698582125763691, + "C_V": 1.5869444305361093, + "loss": 0.06233566254377365 + }, + { + "step": 2400, + "train_acc": 1.0, + "val_acc": 0.31424096655106837, + "gamma": 0.3682383585542208, + "C_V": 1.5828754618775391, + "loss": 0.06663424521684647 + }, + { + "step": 2500, + "train_acc": 1.0, + "val_acc": 0.32475668419286274, + "gamma": 0.34365841551416604, + "C_V": 1.52359689472271, + "loss": 0.08457232266664505 + }, + { + "step": 2600, + "train_acc": 1.0, + "val_acc": 0.34008278330909497, + "gamma": 0.3569614645728618, + "C_V": 1.5551167541393927, + "loss": 0.07498012483119965 + }, + { + "step": 2700, + "train_acc": 0.9997389033942559, + "val_acc": 0.3727486296006265, + "gamma": 0.35398873736177283, + "C_V": 1.5479606282963678, + "loss": 0.09010390937328339 + }, + { + "step": 2800, + "train_acc": 1.0, + "val_acc": 0.4323749860163329, + "gamma": 0.3523303811576417, + "C_V": 1.5439970795409477, + "loss": 0.048116497695446014 + }, + { + "step": 2900, + "train_acc": 1.0, + "val_acc": 0.46313905358541224, + "gamma": 0.35048456167218756, + "C_V": 1.5396092856153127, + "loss": 0.0807647556066513 + }, + { + "step": 3000, + "train_acc": 1.0, + "val_acc": 0.5722116567848753, + "gamma": 0.3433820120676047, + "C_V": 1.5229555363673026, + "loss": 0.048496004194021225 + }, + { + "step": 3100, + "train_acc": 1.0, + "val_acc": 0.6514151471081776, + "gamma": 0.34452075692686557, + "C_V": 1.5256013223418365, + "loss": 0.05813753604888916 + }, + { + "step": 3200, + "train_acc": 1.0, + "val_acc": 0.7506432486855353, + "gamma": 0.34386468457120734, + "C_V": 1.524075867409282, + "loss": 0.049131039530038834 + }, + { + "step": 3300, + "train_acc": 1.0, + "val_acc": 0.8490882649065891, + "gamma": 0.3478733168418804, + "C_V": 1.5334443840837784, + "loss": 0.04255269840359688 + }, + { + "step": 3400, + "train_acc": 1.0, + "val_acc": 0.9258306298243651, + "gamma": 0.3430918629810425, + "C_V": 1.5222828636862222, + "loss": 0.04282087832689285 + }, + { + "step": 3500, + "train_acc": 1.0, + "val_acc": 0.9739344445687437, + "gamma": 0.348926551503103, + "C_V": 1.5359250209153108, + "loss": 0.03849276900291443 + }, + { + "step": 3600, + "train_acc": 1.0, + "val_acc": 0.9933997091397248, + "gamma": 0.3412478651579237, + "C_V": 1.5180216459408236, + "loss": 0.03529882803559303 + }, + { + "step": 3700, + "train_acc": 1.0, + "val_acc": 0.9988813066338517, + "gamma": 0.34073233771507366, + "C_V": 1.5168345987639447, + "loss": 0.029863839969038963 + }, + { + "step": 3800, + "train_acc": 1.0, + "val_acc": 0.9997762613267703, + "gamma": 0.33944189098130095, + "C_V": 1.5138713556715901, + "loss": 0.027958324179053307 + }, + { + "step": 3900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3418420939767, + "C_V": 1.5193922170473755, + "loss": 0.025837423279881477 + }, + { + "step": 4000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33710938816026687, + "C_V": 1.5085445202258647, + "loss": 0.02465984970331192 + }, + { + "step": 4100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3404752380104295, + "C_V": 1.5162432976486386, + "loss": 0.022582512348890305 + }, + { + "step": 4200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3378402082801355, + "C_V": 1.510209487958555, + "loss": 0.021168529987335205 + }, + { + "step": 4300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3394515867994486, + "C_V": 1.513893576936633, + "loss": 0.019745897501707077 + }, + { + "step": 4400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33900355017016615, + "C_V": 1.5128674295564504, + "loss": 0.018549907952547073 + }, + { + "step": 4500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3376258071216251, + "C_V": 1.5097206545056625, + "loss": 0.017805520445108414 + }, + { + "step": 4600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3392767144986131, + "C_V": 1.5134928977736186, + "loss": 0.01685013435781002 + }, + { + "step": 4700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3365984666824926, + "C_V": 1.5073827083263536, + "loss": 0.015480736270546913 + }, + { + "step": 4800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3368757198486693, + "C_V": 1.508012947093102, + "loss": 0.015223415568470955 + }, + { + "step": 4900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.337828678443265, + "C_V": 1.5101831919404858, + "loss": 0.014560554176568985 + }, + { + "step": 5000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3359600427248459, + "C_V": 1.5059334744003006, + "loss": 0.014130224473774433 + }, + { + "step": 5100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33487479379837404, + "C_V": 1.5034763239702873, + "loss": 0.0130455382168293 + }, + { + "step": 5200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3326544390725228, + "C_V": 1.498474041859512, + "loss": 0.012629157863557339 + }, + { + "step": 5300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3349150892681535, + "C_V": 1.5035674150231726, + "loss": 0.011814754456281662 + }, + { + "step": 5400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33281258388149726, + "C_V": 1.4988292282514881, + "loss": 0.011429711245000362 + }, + { + "step": 5500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3330667432092071, + "C_V": 1.4994004119870803, + "loss": 0.010795604437589645 + }, + { + "step": 5600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33117765541457256, + "C_V": 1.495165357580651, + "loss": 0.01032460667192936 + }, + { + "step": 5700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33174319020528414, + "C_V": 1.496430691528895, + "loss": 0.0099105229601264 + }, + { + "step": 5800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33209745697594445, + "C_V": 1.4972244235997518, + "loss": 0.009713426232337952 + }, + { + "step": 5900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33453355480960956, + "C_V": 1.502705368884376, + "loss": 0.009087443351745605 + }, + { + "step": 6000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3331675489253196, + "C_V": 1.4996270778189937, + "loss": 0.008833050727844238 + }, + { + "step": 6100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3332682829701635, + "C_V": 1.4998536509629548, + "loss": 0.008512184023857117 + }, + { + "step": 6200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.331825538478602, + "C_V": 1.496615117140294, + "loss": 0.008380085229873657 + }, + { + "step": 6300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3318277789221522, + "C_V": 1.4966201354298616, + "loss": 0.008024008013308048 + }, + { + "step": 6400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33060475264661604, + "C_V": 1.4938857184208312, + "loss": 0.007714222185313702 + }, + { + "step": 6500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33468925638390923, + "C_V": 1.5030570445395324, + "loss": 0.0076011670753359795 + }, + { + "step": 6600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33863775138366814, + "C_V": 1.5120306641211358, + "loss": 0.007343090604990721 + }, + { + "step": 6700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3362673528770654, + "C_V": 1.5066307259928755, + "loss": 0.006820996291935444 + }, + { + "step": 6800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3358291931549203, + "C_V": 1.5056367875459087, + "loss": 0.006641845218837261 + }, + { + "step": 6900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3349659440521665, + "C_V": 1.5036823919863163, + "loss": 0.006390230264514685 + }, + { + "step": 7000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33531825423780387, + "C_V": 1.504479408943737, + "loss": 0.006220953073352575 + }, + { + "step": 7100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33606169681353915, + "C_V": 1.5061640444611604, + "loss": 0.00607119919732213 + }, + { + "step": 7200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33839794395358, + "C_V": 1.5114826062902034, + "loss": 0.005732213146984577 + }, + { + "step": 7300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33450972882485686, + "C_V": 1.502651568796294, + "loss": 0.00570358382537961 + }, + { + "step": 7400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33579156972245383, + "C_V": 1.5055515022327253, + "loss": 0.005453728139400482 + }, + { + "step": 7500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3339366782545507, + "C_V": 1.501358755770329, + "loss": 0.005223960615694523 + }, + { + "step": 7600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.338132841941396, + "C_V": 1.5108772022065742, + "loss": 0.0049489219672977924 + }, + { + "step": 7700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33342639074164176, + "C_V": 1.5002094083991981, + "loss": 0.005055162124335766 + }, + { + "step": 7800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33184962932661033, + "C_V": 1.4966690791358217, + "loss": 0.004769342020153999 + }, + { + "step": 7900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3349055969515583, + "C_V": 1.5035459559072033, + "loss": 0.0046088979579508305 + }, + { + "step": 8000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33581641259060707, + "C_V": 1.5056078153036547, + "loss": 0.004394772462546825 + }, + { + "step": 8100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.330244670997799, + "C_V": 1.49308255820793, + "loss": 0.004382298327982426 + }, + { + "step": 8200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.32835211345842735, + "C_V": 1.4888753765744238, + "loss": 0.004154183901846409 + }, + { + "step": 8300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33718398001774585, + "C_V": 1.5087142885091605, + "loss": 0.004041675478219986 + }, + { + "step": 8400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3386322116895701, + "C_V": 1.5120179991751042, + "loss": 0.0037901494652032852 + }, + { + "step": 8500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3324343173921188, + "C_V": 1.4979799382338623, + "loss": 0.003687863703817129 + }, + { + "step": 8600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3384522322623374, + "C_V": 1.5116066424345502, + "loss": 0.0035243190359324217 + }, + { + "step": 8700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3358728741072544, + "C_V": 1.505735816250181, + "loss": 0.0034888572990894318 + }, + { + "step": 8800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3407879362154756, + "C_V": 1.5169625298709162, + "loss": 0.0033711749128997326 + }, + { + "step": 8900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33357154526139987, + "C_V": 1.5005361684207197, + "loss": 0.00330185703933239 + }, + { + "step": 9000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33084054932690404, + "C_V": 1.4944121300149271, + "loss": 0.0031605560798197985 + }, + { + "step": 9100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33842972211611727, + "C_V": 1.5115552095215463, + "loss": 0.0030807883013039827 + }, + { + "step": 9200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3315602274973029, + "C_V": 1.4960210944000418, + "loss": 0.00293061975389719 + }, + { + "step": 9300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3332504448619245, + "C_V": 1.4998135241243804, + "loss": 0.0029069366864860058 + }, + { + "step": 9400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33812260403024214, + "C_V": 1.5108538319772011, + "loss": 0.002828003838658333 + }, + { + "step": 9500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.335664325929844, + "C_V": 1.5052631358381587, + "loss": 0.0027258049231022596 + }, + { + "step": 9600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33655164950667976, + "C_V": 1.507276337723095, + "loss": 0.0026471822056919336 + }, + { + "step": 9700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33577616403962174, + "C_V": 1.5055165832074282, + "loss": 0.002526696305721998 + }, + { + "step": 9800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3328784293733482, + "C_V": 1.4989771640282346, + "loss": 0.0024728039279580116 + }, + { + "step": 9900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33375031169421404, + "C_V": 1.5009387884937124, + "loss": 0.0023489920422434807 + }, + { + "step": 10000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3277899937825021, + "C_V": 1.4876303398501385, + "loss": 0.002287168288603425 + }, + { + "step": 10100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3294927477281451, + "C_V": 1.4914081788254145, + "loss": 0.002185388933867216 + }, + { + "step": 10200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33410073837777177, + "C_V": 1.5017286512134784, + "loss": 0.002161257201805711 + }, + { + "step": 10300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33337213933321636, + "C_V": 1.5000873185824641, + "loss": 0.002103738021105528 + }, + { + "step": 10400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3285727937179978, + "C_V": 1.489364730299588, + "loss": 0.0020630364306271076 + }, + { + "step": 10500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3298351139971996, + "C_V": 1.4921700925939312, + "loss": 0.002003427827730775 + }, + { + "step": 10600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33059881561760734, + "C_V": 1.4938724689031238, + "loss": 0.001908608479425311 + }, + { + "step": 10700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3342599219588108, + "C_V": 1.5020877261022134, + "loss": 0.001801896607503295 + }, + { + "step": 10800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.32631375089634745, + "C_V": 1.4843705082751972, + "loss": 0.001830499735660851 + }, + { + "step": 10900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3392134639265806, + "C_V": 1.5133480260392154, + "loss": 0.0017095296643674374 + }, + { + "step": 11000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.32161064599369954, + "C_V": 1.4740797362081137, + "loss": 0.001686910749413073 + }, + { + "step": 11100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3374945810448124, + "C_V": 1.5094216158670257, + "loss": 0.0015850039198994637 + }, + { + "step": 11200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3231202562755106, + "C_V": 1.4773673008702568, + "loss": 0.0015551485121250153 + }, + { + "step": 11300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3293766350011919, + "C_V": 1.4911499541948965, + "loss": 0.001543126069009304 + }, + { + "step": 11400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33359805637406476, + "C_V": 1.5005958634498222, + "loss": 0.0015364409191533923 + }, + { + "step": 11500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3260460118726725, + "C_V": 1.48378081829983, + "loss": 0.0014649470103904605 + }, + { + "step": 11600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3338209326948094, + "C_V": 1.5010979015674761, + "loss": 0.0013820375315845013 + }, + { + "step": 11700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.32743507027109275, + "C_V": 1.4868452929936042, + "loss": 0.0014197693672031164 + }, + { + "step": 11800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3195920854580409, + "C_V": 1.4697065960397975, + "loss": 0.0013384540798142552 + }, + { + "step": 11900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33104688277630023, + "C_V": 1.4948730699547623, + "loss": 0.0012470607180148363 + }, + { + "step": 12000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.322039358586503, + "C_V": 1.4750118796204381, + "loss": 0.0012155169388279319 + }, + { + "step": 12100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3276952690229574, + "C_V": 1.487420739322668, + "loss": 0.0011803358793258667 + }, + { + "step": 12200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33523749274392295, + "C_V": 1.50429663087901, + "loss": 0.0011143480660393834 + }, + { + "step": 12300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33239716886626214, + "C_V": 1.4978965836645388, + "loss": 0.0010899719782173634 + }, + { + "step": 12400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3319354517327769, + "C_V": 1.4968613475954182, + "loss": 0.0010891229612752795 + }, + { + "step": 12500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.32454367328337375, + "C_V": 1.4804806179860233, + "loss": 0.0010638745734468102 + }, + { + "step": 12600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.32785096235293987, + "C_V": 1.487765278219578, + "loss": 0.0010355892591178417 + }, + { + "step": 12700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3318104679255295, + "C_V": 1.496581362020722, + "loss": 0.0009984886273741722 + }, + { + "step": 12800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3213200007959767, + "C_V": 1.4734484605010176, + "loss": 0.0009895828552544117 + }, + { + "step": 12900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33362873042883207, + "C_V": 1.5006649380960457, + "loss": 0.0009339888347312808 + }, + { + "step": 13000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.326977913354042, + "C_V": 1.4858353386046426, + "loss": 0.0009216577745974064 + }, + { + "step": 13100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3339009308284876, + "C_V": 1.5012781826039636, + "loss": 0.0009290777379646897 + }, + { + "step": 13200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33909511312387625, + "C_V": 1.513077024935714, + "loss": 0.0008745994418859482 + }, + { + "step": 13300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3263910774608825, + "C_V": 1.4845409057685521, + "loss": 0.0008127463515847921 + }, + { + "step": 13400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.32907545543333755, + "C_V": 1.49048057355821, + "loss": 0.0008269792888313532 + }, + { + "step": 13500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3389221991029849, + "C_V": 1.5126812587612262, + "loss": 0.0007734993705525994 + }, + { + "step": 13600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33368704054283826, + "C_V": 1.5007962636876966, + "loss": 0.0007586776046082377 + }, + { + "step": 13700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3409574139909175, + "C_V": 1.5173526282354972, + "loss": 0.000709090381860733 + }, + { + "step": 13800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3324797158542495, + "C_V": 1.4980818167641687, + "loss": 0.0006850300705991685 + }, + { + "step": 13900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3391606185938484, + "C_V": 1.5132270081607022, + "loss": 0.0006886720657348633 + }, + { + "step": 14000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33940509183212725, + "C_V": 1.5137870238410562, + "loss": 0.0006752238841727376 + }, + { + "step": 14100, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3370058373183229, + "C_V": 1.5083089057001686, + "loss": 0.0006267992430366576 + }, + { + "step": 14200, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3349959107342462, + "C_V": 1.5037501515278242, + "loss": 0.0006317031220532954 + }, + { + "step": 14300, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.34291548456461374, + "C_V": 1.5218742437377282, + "loss": 0.0006018645945005119 + }, + { + "step": 14400, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.34134522431502, + "C_V": 1.5182460325441836, + "loss": 0.0005817135097458959 + }, + { + "step": 14500, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33874896807924104, + "C_V": 1.5122849745810831, + "loss": 0.000568479357752949 + }, + { + "step": 14600, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3346192029386878, + "C_V": 1.5028987978260726, + "loss": 0.0005473577766679227 + }, + { + "step": 14700, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.33931869028630945, + "C_V": 1.5135890561719612, + "loss": 0.0005187753704376519 + }, + { + "step": 14800, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.34028756454290476, + "C_V": 1.5158119602628524, + "loss": 0.0005409690202213824 + }, + { + "step": 14900, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.34302853079047946, + "C_V": 1.5221361152916082, + "loss": 0.0005100182024762034 + }, + { + "step": 15000, + "train_acc": 1.0, + "val_acc": 1.0, + "gamma": 0.3396325653786703, + "C_V": 1.5143084706674301, + "loss": 0.00048082906869240105 + } + ] +} \ No newline at end of file diff --git a/data/exp_d4/area_law_results.json b/data/exp_d4/area_law_results.json new file mode 100644 index 0000000000000000000000000000000000000000..c3846e3f26f838d4302c265906a6c082168140b4 --- /dev/null +++ b/data/exp_d4/area_law_results.json @@ -0,0 +1,568 @@ +{ + "N_context": 2000, + "n_models": 56, + "results": [ + { + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "mongo", + "gamma": 0.2870574377368437, + "S_gamma": 7.540761817121435, + "area_law_pred": 5.419006875018258, + "ratio_S_area": 1.3915394445953768, + "phase": "A", + "r2": 0.814928 + }, + { + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "mongo", + "gamma": 0.2870574377368437, + "S_gamma": 7.540761817121435, + "area_law_pred": 5.419006875018258, + "ratio_S_area": 1.3915394445953768, + "phase": "A", + "r2": 0.814928 + }, + { + "model": "EleutherAI/pythia-160m", + "corpus": "mongo", + "gamma": 0.510891565491545, + "S_gamma": 7.330756084048007, + "area_law_pred": 3.7176655028380927, + "ratio_S_area": 1.971870809369927, + "phase": "A", + "r2": 0.916873 + }, + { + "model": "allenai/OLMo-7B", + "corpus": "mongo", + "gamma": 0.5243966907611701, + "S_gamma": 7.310088552158613, + "area_law_pred": 3.6150143629597755, + "ratio_S_area": 2.0221464752835763, + "phase": "A", + "r2": 0.976863 + }, + { + "model": "allenai/OLMo-7B", + "corpus": "random", + "gamma": 0.5411598042384096, + "S_gamma": 7.282815329848365, + "area_law_pred": 3.487599572501043, + "ratio_S_area": 2.0882028393602767, + "phase": "A", + "r2": 0.96368 + }, + { + "model": "allenai/OLMo-7B-hf", + "corpus": "random", + "gamma": 0.550891782676269, + "S_gamma": 7.266127883242827, + "area_law_pred": 3.4136277536565065, + "ratio_S_area": 2.1285648019060415, + "phase": "A", + "r2": 0.957841 + }, + { + "model": "HuggingFaceTB/SmolLM2-135M", + "corpus": "random", + "gamma": 0.62665506839462, + "S_gamma": 7.112817971850239, + "area_law_pred": 2.837758408896903, + "ratio_S_area": 2.506491725846085, + "phase": "A", + "r2": 0.828515 + }, + { + "model": "google/gemma-2-9b-it", + "corpus": "mongo", + "gamma": 0.6276459084140061, + "S_gamma": 7.110519287154698, + "area_law_pred": 2.8302271305565383, + "ratio_S_area": 2.5123493483565325, + "phase": "A", + "r2": 0.977314 + }, + { + "model": "microsoft/phi-3-mini-4k-instruct", + "corpus": "mongo", + "gamma": 0.6295631683122206, + "S_gamma": 7.106048616532607, + "area_law_pred": 2.815654225080619, + "ratio_S_area": 2.5237646559137223, + "phase": "A", + "r2": 0.985314 + }, + { + "model": "EleutherAI/pythia-2.8b", + "corpus": "mongo", + "gamma": 0.6741618914822415, + "S_gamma": 6.993335911568815, + "area_law_pred": 2.4766636804451707, + "ratio_S_area": 2.8236921980104257, + "phase": "A", + "r2": 0.999287 + }, + { + "model": "EleutherAI/pythia-14m", + "corpus": "mongo", + "gamma": 0.6852875452461592, + "S_gamma": 6.9625275628284164, + "area_law_pred": 2.3920986713869947, + "ratio_S_area": 2.9106356046723527, + "phase": "A", + "r2": 0.904735 + }, + { + "model": "EleutherAI/pythia-1.4b", + "corpus": "random", + "gamma": 0.6875846707476845, + "S_gamma": 6.956028732948211, + "area_law_pred": 2.3746384445125743, + "ratio_S_area": 2.9293001421005913, + "phase": "A", + "r2": 0.948837 + }, + { + "model": "EleutherAI/pythia-1.4b", + "corpus": "mongo", + "gamma": 0.7050725013322717, + "S_gamma": 6.904986546637838, + "area_law_pred": 2.2417151500101298, + "ratio_S_area": 3.080224776375641, + "phase": "A", + "r2": 0.841258 + }, + { + "model": "EleutherAI/pythia-1b", + "corpus": "random", + "gamma": 0.7127537531683527, + "S_gamma": 6.881681916795565, + "area_law_pred": 2.1833307040369005, + "ratio_S_area": 3.151919177462022, + "phase": "A", + "r2": 0.956201 + }, + { + "model": "gpt2-large", + "corpus": "random", + "gamma": 0.7274410953080044, + "S_gamma": 6.835595893390991, + "area_law_pred": 2.0716936490434854, + "ratio_S_area": 3.2995206103696995, + "phase": "A", + "r2": 0.977312 + }, + { + "model": "gpt2-medium", + "corpus": "random", + "gamma": 0.7409610966104535, + "S_gamma": 6.791383942293905, + "area_law_pred": 1.9689294378906879, + "ratio_S_area": 3.4492774660170187, + "phase": "A", + "r2": 0.9987 + }, + { + "model": "EleutherAI/pythia-70m", + "corpus": "mongo", + "gamma": 0.7476017873166874, + "S_gamma": 6.769034916681587, + "area_law_pred": 1.9184541955686165, + "ratio_S_area": 3.528379740479179, + "phase": "A", + "r2": 0.984269 + }, + { + "model": "HuggingFaceTB/SmolLM2-135M", + "corpus": "mongo", + "gamma": 0.7479961324735662, + "S_gamma": 6.76769458718171, + "area_law_pred": 1.9154568164957875, + "ratio_S_area": 3.533201338134471, + "phase": "A", + "r2": 0.988937 + }, + { + "model": "gpt2-large", + "corpus": "mongo", + "gamma": 0.7526597997629645, + "S_gamma": 6.751731138994513, + "area_law_pred": 1.8800087363253144, + "ratio_S_area": 3.5913296616863177, + "phase": "A", + "r2": 0.963281 + }, + { + "model": "meta-llama/Meta-Llama-3-8B", + "corpus": "random", + "gamma": 0.7589145044527899, + "S_gamma": 6.7299962332100405, + "area_law_pred": 1.8324673360647112, + "ratio_S_area": 3.672641853285607, + "phase": "A", + "r2": 0.984284 + }, + { + "model": "Qwen/Qwen2.5-3B", + "corpus": "mongo", + "gamma": 0.7720333740995954, + "S_gamma": 6.683194212515172, + "area_law_pred": 1.7327520874998952, + "ratio_S_area": 3.8569823465964093, + "phase": "A", + "r2": 0.995841 + }, + { + "model": "gpt2-medium", + "corpus": "mongo", + "gamma": 0.7842197184521328, + "S_gamma": 6.6382427776423985, + "area_law_pred": 1.6401248727378668, + "ratio_S_area": 4.047400833914039, + "phase": "A", + "r2": 0.996208 + }, + { + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "random", + "gamma": 0.8266242679750889, + "S_gamma": 6.470778115399689, + "area_law_pred": 1.3178120279730556, + "ratio_S_area": 4.9102436296263585, + "phase": "A", + "r2": 0.993628 + }, + { + "model": "Qwen/Qwen2.5-7B", + "corpus": "random", + "gamma": 0.8270155146179213, + "S_gamma": 6.469153732984772, + "area_law_pred": 1.314838200403263, + "ratio_S_area": 4.9201139204813735, + "phase": "A", + "r2": 0.984663 + }, + { + "model": "allenai/OLMo-7B-hf", + "corpus": "mongo", + "gamma": 0.8291935914574181, + "S_gamma": 6.460084559322255, + "area_law_pred": 1.2982828507968605, + "ratio_S_area": 4.975868359777827, + "phase": "A", + "r2": 0.99587 + }, + { + "model": "mistralai/Mistral-7B-v0.1", + "corpus": "random", + "gamma": 0.8296009929924347, + "S_gamma": 6.4583832741852625, + "area_law_pred": 1.2951862314673315, + "ratio_S_area": 4.986451459469643, + "phase": "A", + "r2": 0.996923 + }, + { + "model": "EleutherAI/gpt-j-6B", + "corpus": "random", + "gamma": 0.8347553760849855, + "S_gamma": 6.436725078705287, + "area_law_pred": 1.25600826834174, + "ratio_S_area": 5.124747377024397, + "phase": "A", + "r2": 0.980141 + }, + { + "model": "microsoft/phi-2", + "corpus": "random", + "gamma": 0.8707349253167322, + "S_gamma": 6.278751523867516, + "area_law_pred": 0.9825312240929408, + "ratio_S_area": 6.3903837047662, + "phase": "A", + "r2": 0.947956 + }, + { + "model": "bigscience/bloom-7b1", + "corpus": "random", + "gamma": 0.882390442263598, + "S_gamma": 6.225096260313582, + "area_law_pred": 0.8939387766642747, + "ratio_S_area": 6.963671811555685, + "phase": "A", + "r2": 0.994632 + }, + { + "model": "EleutherAI/gpt-j-6B", + "corpus": "mongo", + "gamma": 0.8967966648253523, + "S_gamma": 6.157167619017644, + "area_law_pred": 0.784438484161926, + "ratio_S_area": 7.849140172662238, + "phase": "A", + "r2": 0.98687 + }, + { + "model": "deepseek-ai/deepseek-llm-7b-base", + "corpus": "random", + "gamma": 0.9103448618583042, + "S_gamma": 6.09171169898598, + "area_law_pred": 0.6814599600118006, + "ratio_S_area": 8.939207079577342, + "phase": "A", + "r2": 0.992208 + }, + { + "model": "Qwen/Qwen2.5-0.5B", + "corpus": "random", + "gamma": 0.9194920254637005, + "S_gamma": 6.046685882415325, + "area_law_pred": 0.6119332616657105, + "ratio_S_area": 9.881283239868296, + "phase": "A", + "r2": 0.995818 + }, + { + "model": "HuggingFaceTB/SmolLM2-360M", + "corpus": "random", + "gamma": 0.9197098362636822, + "S_gamma": 6.0456057444940505, + "area_law_pred": 0.6102777030204145, + "ratio_S_area": 9.90631922905402, + "phase": "A", + "r2": 0.992716 + }, + { + "model": "EleutherAI/pythia-1b", + "corpus": "mongo", + "gamma": 0.9311078627189842, + "S_gamma": 5.988575565494448, + "area_law_pred": 0.5236424157023841, + "ratio_S_area": 11.436383657847337, + "phase": "A", + "r2": 0.983104 + }, + { + "model": "EleutherAI/pythia-410m", + "corpus": "random", + "gamma": 0.9362347719713631, + "S_gamma": 5.962605278027159, + "area_law_pred": 0.4846732785561282, + "ratio_S_area": 12.302318988556848, + "phase": "A", + "r2": 0.987529 + }, + { + "model": "deepseek-ai/deepseek-llm-7b-base", + "corpus": "mongo", + "gamma": 0.9469730981988765, + "S_gamma": 5.907593587772118, + "area_law_pred": 0.4030523083220558, + "ratio_S_area": 14.657138703326074, + "phase": "A", + "r2": 0.995256 + }, + { + "model": "Qwen/Qwen2.5-3B", + "corpus": "random", + "gamma": 0.9643628833140916, + "S_gamma": 5.816818113694677, + "area_law_pred": 0.2708742478689094, + "ratio_S_area": 21.47423817309406, + "phase": "A", + "r2": 0.97891 + }, + { + "model": "HuggingFaceTB/SmolLM2-360M", + "corpus": "mongo", + "gamma": 0.9691725803316233, + "S_gamma": 5.791361269838827, + "area_law_pred": 0.23431620997870006, + "ratio_S_area": 24.716007784375126, + "phase": "A", + "r2": 0.998139 + }, + { + "model": "Qwen/Qwen2.5-7B", + "corpus": "mongo", + "gamma": 0.9966953735480816, + "S_gamma": 5.643021626136776, + "area_law_pred": 0.025118143326254574, + "ratio_S_area": 224.6591857065504, + "phase": "A", + "r2": 0.993942 + }, + { + "model": "EleutherAI/pythia-14m", + "corpus": "random", + "gamma": 1.003714187534367, + "S_gamma": 9.113816929580823, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.977698 + }, + { + "model": "gpt2-xl", + "corpus": "mongo", + "gamma": 1.0097027213131609, + "S_gamma": 8.144856340401692, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.980958 + }, + { + "model": "EleutherAI/pythia-160m", + "corpus": "random", + "gamma": 1.0171452847779678, + "S_gamma": 7.564188500805029, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.981723 + }, + { + "model": "EleutherAI/pythia-410m", + "corpus": "mongo", + "gamma": 1.0218530106365162, + "S_gamma": 7.314101726492739, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.981594 + }, + { + "model": "gpt2", + "corpus": "mongo", + "gamma": 1.0231174997210821, + "S_gamma": 7.255805984306381, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.955115 + }, + { + "model": "gpt2-xl", + "corpus": "random", + "gamma": 1.024099984346949, + "S_gamma": 7.2125851754934445, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.988966 + }, + { + "model": "Qwen/Qwen2.5-0.5B", + "corpus": "mongo", + "gamma": 1.0283740139174087, + "S_gamma": 7.042254100757362, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.996775 + }, + { + "model": "microsoft/phi-3-mini-4k-instruct", + "corpus": "random", + "gamma": 1.0366024777115062, + "S_gamma": 6.7735078789401655, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.977923 + }, + { + "model": "microsoft/phi-2", + "corpus": "mongo", + "gamma": 1.0446992618601783, + "S_gamma": 6.55917099187236, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.979964 + }, + { + "model": "meta-llama/Meta-Llama-3-8B", + "corpus": "mongo", + "gamma": 1.0454762537473639, + "S_gamma": 6.540515247430771, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.997461 + }, + { + "model": "gpt2", + "corpus": "random", + "gamma": 1.0514830126336872, + "S_gamma": 6.405275999206491, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.907872 + }, + { + "model": "mistralai/Mistral-7B-v0.1", + "corpus": "mongo", + "gamma": 1.060750419523944, + "S_gamma": 6.2218961952351, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.99869 + }, + { + "model": "google/gemma-2-9b-it", + "corpus": "random", + "gamma": 1.1347958464287666, + "S_gamma": 5.259629681982037, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.976472 + }, + { + "model": "EleutherAI/pythia-70m", + "corpus": "random", + "gamma": 1.1705141984482668, + "S_gamma": 4.933980978257978, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.994076 + }, + { + "model": "bigscience/bloom-7b1", + "corpus": "mongo", + "gamma": 1.2178239236386044, + "S_gamma": 4.562721586731967, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.99733 + }, + { + "model": "EleutherAI/pythia-31m", + "corpus": "mongo", + "gamma": 1.2350013988825523, + "S_gamma": 4.439867464810442, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.973742 + }, + { + "model": "EleutherAI/pythia-31m", + "corpus": "random", + "gamma": 1.5398244746231446, + "S_gamma": 2.8462032124350634, + "area_law_pred": null, + "ratio_S_area": null, + "phase": "B", + "r2": 0.964259 + } + ], + "phase_a_ratio_mean": 11.448812263895999, + "phase_a_ratio_std": 34.96120529181214 +} \ No newline at end of file diff --git a/data/exp_e_entropy/EleutherAI--pythia-1b_T1000.json b/data/exp_e_entropy/EleutherAI--pythia-1b_T1000.json new file mode 100644 index 0000000000000000000000000000000000000000..bccc9a2e1c0604a01fa7a94574cc02261479c9ad --- /dev/null +++ b/data/exp_e_entropy/EleutherAI--pythia-1b_T1000.json @@ -0,0 +1,48 @@ +{ + "model": "EleutherAI/pythia-1b", + "T_seq": 1000, + "n_layers": 16, + "n_samples": 5, + "H_attn": [ + 4.388372898101807, + 3.8320794105529785, + 4.055876731872559, + 3.7340807914733887, + 3.418897867202759, + 3.2817041873931885, + 2.9064135551452637, + 2.2784643173217773, + 2.7913360595703125, + 3.059999704360962, + 2.1062607765197754, + 3.629648447036743, + 2.8487510681152344, + 3.1419475078582764, + 4.2374043464660645, + 3.3004088401794434 + ], + "ln_T": 6.907755278982137, + "R_profile": [ + 0.0, + 0.0548, + 0.0479, + -0.0083, + -0.0103, + 0.0232, + 0.08310000000000001, + 0.0781, + 0.2341, + 0.2519, + 0.32630000000000003, + 0.3444, + 0.5439, + 0.6936, + 0.7456999999999999, + 0.9076000000000001 + ], + "pearson_r": 0.07496415288279364, + "ols_slope": 0.16286534377832787, + "slope_ratio_vs_lnT": 0.0235771733654013, + "confirmed_r_gt_090": false, + "confirmed_slope_ratio": false +} \ No newline at end of file diff --git a/data/exp_e_entropy/EleutherAI--pythia-70m_T500.json b/data/exp_e_entropy/EleutherAI--pythia-70m_T500.json new file mode 100644 index 0000000000000000000000000000000000000000..c53e5b81b35a41dbfced86f84b921d573b92eec7 --- /dev/null +++ b/data/exp_e_entropy/EleutherAI--pythia-70m_T500.json @@ -0,0 +1,28 @@ +{ + "model": "EleutherAI/pythia-70m", + "T_seq": 500, + "n_layers": 6, + "n_samples": 3, + "H_attn": [ + 2.931483745574951, + 2.6581687927246094, + 2.846722364425659, + 3.681366443634033, + 2.142681837081909, + 0.6738190054893494 + ], + "ln_T": 6.214608098422191, + "R_profile": [ + 0.0, + -1.6746, + -0.7017, + -0.6383, + 1.2949000000000002, + 0.9589 + ], + "pearson_r": 0.614123146616274, + "ols_slope": 0.5616460465455614, + "slope_ratio_vs_lnT": 0.0903751351091883, + "confirmed_r_gt_090": false, + "confirmed_slope_ratio": false +} \ No newline at end of file diff --git a/data/exp_e_entropy/gpt2-large_T200.json b/data/exp_e_entropy/gpt2-large_T200.json new file mode 100644 index 0000000000000000000000000000000000000000..922d371bf85319f05037eb2782d3be0bc0d3a702 --- /dev/null +++ b/data/exp_e_entropy/gpt2-large_T200.json @@ -0,0 +1,88 @@ +{ + "model": "gpt2-large", + "T_seq": 200, + "n_layers": 36, + "n_samples": 5, + "H_attn": [ + 2.985027551651001, + 3.6969990730285645, + 3.4595837593078613, + 3.0505709648132324, + 2.7797834873199463, + 2.588606119155884, + 2.6586735248565674, + 2.4294605255126953, + 2.2396984100341797, + 2.0682716369628906, + 1.9441800117492676, + 1.7711260318756104, + 1.5649975538253784, + 1.5639277696609497, + 1.587461233139038, + 1.6603503227233887, + 1.4181594848632812, + 1.6120234727859497, + 1.8441671133041382, + 1.9064106941223145, + 1.885192632675171, + 1.7851909399032593, + 1.924445390701294, + 1.9209076166152954, + 1.9022446870803833, + 1.9883060455322266, + 1.8982183933258057, + 2.2269718647003174, + 2.414278030395508, + 1.905351161956787, + 2.159285068511963, + 2.611708879470825, + 2.483022689819336, + 2.6407902240753174, + 2.8586182594299316, + 3.781723737716675 + ], + "ln_T": 5.298317366548036, + "R_profile": [ + 0.0073, + -0.0321, + -0.0458, + -0.0558, + 0.0405, + 0.0695, + 0.0172, + -0.0127, + -0.0337, + -0.07980000000000001, + -0.059000000000000004, + -0.0535, + -0.0013, + 0.0098, + 0.0322, + -0.0014000000000000002, + -0.0064, + 0.0013, + 0.10949999999999999, + 0.2256, + 0.26649999999999996, + 0.24559999999999998, + 0.2934, + 0.369, + 0.3933, + 0.45189999999999997, + 0.4512, + 0.5273, + 0.5913, + 0.6424, + 0.7066, + 0.7828, + 0.8429000000000001, + 0.9144, + 0.9763, + 1.0004 + ], + "pearson_r": -0.231888681982478, + "ols_slope": -0.41065783227400743, + "slope_ratio_vs_lnT": -0.07750721669992365, + "confirmed_r_gt_090": false, + "confirmed_slope_ratio": false +} \ No newline at end of file diff --git a/data/exp_e_entropy/gpt2-medium_T200.json b/data/exp_e_entropy/gpt2-medium_T200.json new file mode 100644 index 0000000000000000000000000000000000000000..d73b860d5af20b9bba721744ca7e5b6ac5d4e492 --- /dev/null +++ b/data/exp_e_entropy/gpt2-medium_T200.json @@ -0,0 +1,64 @@ +{ + "model": "gpt2-medium", + "T_seq": 200, + "n_layers": 24, + "n_samples": 5, + "H_attn": [ + 4.093043804168701, + 3.64121150970459, + 2.695664882659912, + 2.164116144180298, + 1.731077790260315, + 1.4492874145507812, + 1.5833061933517456, + 1.6558393239974976, + 2.2106261253356934, + 1.5042644739151, + 2.357913017272949, + 2.1462817192077637, + 1.7097190618515015, + 1.3015670776367188, + 2.0276172161102295, + 1.7667170763015747, + 1.2681903839111328, + 1.59676992893219, + 1.8721853494644165, + 2.1708924770355225, + 2.191699743270874, + 1.9172933101654053, + 2.5054259300231934, + 3.262051582336426 + ], + "ln_T": 5.298317366548036, + "R_profile": [ + 0.0556, + -0.014199999999999999, + -0.1865, + -0.2915, + -0.2553, + -0.2365, + -0.1807, + -0.0839, + -0.032400000000000005, + 0.0124, + 0.0377, + 0.1589, + 0.285, + 0.2439, + 0.251, + 0.2391, + 0.2378, + 0.2857, + 0.3181, + 0.4011, + 0.5207, + 0.6774, + 0.826, + 0.932 + ], + "pearson_r": -0.15201175552005797, + "ols_slope": -0.32658239597659805, + "slope_ratio_vs_lnT": -0.061638888987386814, + "confirmed_r_gt_090": false, + "confirmed_slope_ratio": false +} \ No newline at end of file diff --git a/data/exp_e_entropy/gpt2-medium_T500.json b/data/exp_e_entropy/gpt2-medium_T500.json new file mode 100644 index 0000000000000000000000000000000000000000..98666c0da36abde1d1c863d03667584758d6ba13 --- /dev/null +++ b/data/exp_e_entropy/gpt2-medium_T500.json @@ -0,0 +1,64 @@ +{ + "model": "gpt2-medium", + "T_seq": 500, + "n_layers": 24, + "n_samples": 3, + "H_attn": [ + 4.911882400512695, + 4.366139888763428, + 2.929659605026245, + 2.336615562438965, + 1.9231513738632202, + 1.5929408073425293, + 1.952513575553894, + 1.9363460540771484, + 2.57356858253479, + 1.8593672513961792, + 2.678025722503662, + 2.634000062942505, + 1.9815924167633057, + 1.5687710046768188, + 2.433032274246216, + 2.143284559249878, + 1.6216562986373901, + 2.024707317352295, + 2.404418468475342, + 2.787477731704712, + 2.8182625770568848, + 2.4037156105041504, + 3.1506476402282715, + 4.003361225128174 + ], + "ln_T": 6.214608098422191, + "R_profile": [ + 0.0556, + -0.014199999999999999, + -0.1865, + -0.2915, + -0.2553, + -0.2365, + -0.1807, + -0.0839, + -0.032400000000000005, + 0.0124, + 0.0377, + 0.1589, + 0.285, + 0.2439, + 0.251, + 0.2391, + 0.2378, + 0.2857, + 0.3181, + 0.4011, + 0.5207, + 0.6774, + 0.826, + 0.932 + ], + "pearson_r": -0.2642943687624516, + "ols_slope": -0.6837694019728887, + "slope_ratio_vs_lnT": -0.11002614986236846, + "confirmed_r_gt_090": false, + "confirmed_slope_ratio": false +} \ No newline at end of file diff --git a/data/exp_e_entropy/gpt2-xl_T200.json b/data/exp_e_entropy/gpt2-xl_T200.json new file mode 100644 index 0000000000000000000000000000000000000000..e2b4d2ba6537e15f88833ac7f1d897396634cb44 --- /dev/null +++ b/data/exp_e_entropy/gpt2-xl_T200.json @@ -0,0 +1,112 @@ +{ + "model": "gpt2-xl", + "T_seq": 200, + "n_layers": 48, + "n_samples": 5, + "H_attn": [ + 4.1043381690979, + 3.744020700454712, + 3.3394172191619873, + 3.085585117340088, + 2.998553514480591, + 2.958157777786255, + 2.9327144622802734, + 2.8111155033111572, + 2.444821357727051, + 1.9727059602737427, + 2.356365919113159, + 2.0194687843322754, + 1.5285001993179321, + 1.9867370128631592, + 1.7149269580841064, + 1.6522712707519531, + 1.691179633140564, + 1.704365611076355, + 1.4501495361328125, + 1.9920108318328857, + 1.887499213218689, + 2.052227020263672, + 1.4265239238739014, + 1.9933232069015503, + 1.5370559692382812, + 1.678070306777954, + 1.8854421377182007, + 1.6606148481369019, + 1.5124332904815674, + 1.8385456800460815, + 1.6298719644546509, + 1.7575044631958008, + 2.0843191146850586, + 1.97488272190094, + 2.1596264839172363, + 2.1962029933929443, + 1.92404305934906, + 2.1435203552246094, + 1.9544390439987183, + 2.248626947402954, + 2.131019115447998, + 2.171719551086426, + 2.4949898719787598, + 2.351156234741211, + 2.311521291732788, + 2.669712781906128, + 2.7715749740600586, + 3.554456949234009 + ], + "ln_T": 5.298317366548036, + "R_profile": [ + -0.0067, + -0.016, + -0.0062, + 0.0154, + 0.058600000000000006, + 0.0369, + 0.061, + 0.0572, + 0.044500000000000005, + 0.0362, + 0.0329, + 0.0407, + 0.0217, + 0.028399999999999998, + 0.042800000000000005, + 0.0538, + 0.0968, + 0.0941, + 0.0821, + 0.1467, + 0.21100000000000002, + 0.2727, + 0.34, + 0.3619, + 0.3771, + 0.455, + 0.4671, + 0.4542, + 0.4521, + 0.4778, + 0.4768, + 0.461, + 0.5055, + 0.5265, + 0.5713, + 0.612, + 0.6657, + 0.6886, + 0.7204, + 0.7766, + 0.799, + 0.8468000000000001, + 0.877, + 0.9105, + 0.946, + 0.9851000000000001, + 0.998, + 1.0051999999999999 + ], + "pearson_r": 0.0508909500179001, + "ols_slope": 0.0942601902719024, + "slope_ratio_vs_lnT": 0.017790589681741707, + "confirmed_r_gt_090": false, + "confirmed_slope_ratio": false +} \ No newline at end of file diff --git a/data/exp_filt_a/gpt2-medium_filt_comparison.json b/data/exp_filt_a/gpt2-medium_filt_comparison.json new file mode 100644 index 0000000000000000000000000000000000000000..63505eeae7b1da995f3b482b0b1d9442c3c29fb9 --- /dev/null +++ b/data/exp_filt_a/gpt2-medium_filt_comparison.json @@ -0,0 +1,99 @@ +{ + "model": "gpt2-medium", + "theta": null, + "d_head": 64, + "n_dead": 0, + "f_active": 1.0, + "n_prompts": 30, + "baseline": { + "R_curve": [ + -0.0154, + -0.2126, + -0.296, + -0.2649, + -0.2521, + -0.196, + -0.0872, + -0.0299, + 0.0106, + 0.034, + 0.1714, + 0.287, + 0.2542, + 0.262, + 0.2399, + 0.2436, + 0.3028, + 0.3387, + 0.4227, + 0.5352, + 0.6811, + 0.8315, + 0.9307, + 1.0 + ], + "L_crit": 22, + "mu_baseline": -9.7962, + "mu_d10": -5.6337, + "max_R": 1.0 + }, + "filtered": { + "R_curve": [ + -0.0154, + -0.2126, + -0.296, + -0.2649, + -0.2521, + -0.196, + -0.0872, + -0.0299, + 0.0106, + 0.034, + 0.1714, + 0.287, + 0.2542, + 0.262, + 0.2399, + 0.2436, + 0.3028, + 0.3387, + 0.4227, + 0.5352, + 0.6811, + 0.8315, + 0.9307, + 1.0 + ], + "L_crit": 22, + "mu_baseline": -9.7962, + "mu_d10": -5.6337, + "max_R": 1.0 + }, + "delta_R": [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + "lcrit_shift": 0 +} \ No newline at end of file diff --git a/data/exp_filt_a/pythia-70m_filt_comparison.json b/data/exp_filt_a/pythia-70m_filt_comparison.json new file mode 100644 index 0000000000000000000000000000000000000000..f215569ffd16647cad3185fe30d4d49785e8e720 --- /dev/null +++ b/data/exp_filt_a/pythia-70m_filt_comparison.json @@ -0,0 +1,45 @@ +{ + "model": "pythia-70m", + "theta": 10000, + "d_head": 64, + "n_dead": 17, + "f_active": 0.4688, + "n_prompts": 30, + "baseline": { + "R_curve": [ + -0.6006, + -0.2553, + -1.0232, + 1.2353, + 0.9883, + 1.0 + ], + "L_crit": 3, + "mu_baseline": -14.1334, + "mu_d10": -13.5259, + "max_R": 1.2353 + }, + "filtered": { + "R_curve": [ + 0.33, + 0.3474, + 0.3965, + 0.4666, + 0.5993, + 1.0 + ], + "L_crit": 5, + "mu_baseline": -13.0411, + "mu_d10": -13.6504, + "max_R": 1.0 + }, + "delta_R": [ + 0.9306, + 0.6027, + 1.4197, + -0.7687, + -0.389, + 0.0 + ], + "lcrit_shift": 2 +} \ No newline at end of file diff --git a/data/exp_filt_c/pythia-70m_subspace_analysis.json b/data/exp_filt_c/pythia-70m_subspace_analysis.json new file mode 100644 index 0000000000000000000000000000000000000000..c27eb2a0f039aff8ae9528344f90e3e54db44f97 --- /dev/null +++ b/data/exp_filt_c/pythia-70m_subspace_analysis.json @@ -0,0 +1,86 @@ +{ + "model": "pythia-70m", + "theta": 10000, + "n_dead": 17, + "f_active": 0.4688, + "n_prompts": 20, + "per_layer": { + "0": { + "q_dead_energy": 0.7803, + "k_dead_energy": 0.582, + "score_dead_frac": 0.9195, + "score_active_frac": 0.0805, + "cosim_A1_A2_dead": 1.0, + "cosim_A1_A2_active": 1.0, + "cosim_A1_rand_dead": 0.9113, + "cosim_A1_rand_active": 0.8475, + "dead_discrimination": 0.0887, + "active_discrimination": 0.1525 + }, + "1": { + "q_dead_energy": 0.8043, + "k_dead_energy": 0.4436, + "score_dead_frac": 0.8771, + "score_active_frac": 0.1229, + "cosim_A1_A2_dead": 0.991, + "cosim_A1_A2_active": 0.9925, + "cosim_A1_rand_dead": 0.9675, + "cosim_A1_rand_active": 0.9697, + "dead_discrimination": 0.0235, + "active_discrimination": 0.0228 + }, + "2": { + "q_dead_energy": 0.7957, + "k_dead_energy": 0.4095, + "score_dead_frac": -1.0618, + "score_active_frac": 2.0618, + "cosim_A1_A2_dead": 0.9916, + "cosim_A1_A2_active": 0.9951, + "cosim_A1_rand_dead": 0.9852, + "cosim_A1_rand_active": 0.9911, + "dead_discrimination": 0.0064, + "active_discrimination": 0.004 + }, + "3": { + "q_dead_energy": 0.4186, + "k_dead_energy": 0.3957, + "score_dead_frac": 0.416, + "score_active_frac": 0.584, + "cosim_A1_A2_dead": 0.9954, + "cosim_A1_A2_active": 0.9987, + "cosim_A1_rand_dead": 0.9914, + "cosim_A1_rand_active": 0.998, + "dead_discrimination": 0.004, + "active_discrimination": 0.0007 + }, + "4": { + "q_dead_energy": 0.4395, + "k_dead_energy": 0.3844, + "score_dead_frac": 0.3923, + "score_active_frac": 0.6077, + "cosim_A1_A2_dead": 0.9987, + "cosim_A1_A2_active": 0.9999, + "cosim_A1_rand_dead": 0.9978, + "cosim_A1_rand_active": 0.9998, + "dead_discrimination": 0.0009, + "active_discrimination": 0.0001 + }, + "5": { + "q_dead_energy": 0.3863, + "k_dead_energy": 0.3832, + "score_dead_frac": 0.3766, + "score_active_frac": 0.6234, + "cosim_A1_A2_dead": 0.9974, + "cosim_A1_A2_active": 0.9999, + "cosim_A1_rand_dead": 0.995, + "cosim_A1_rand_active": 0.9998, + "dead_discrimination": 0.0024, + "active_discrimination": 0.0001 + } + }, + "summary": { + "mean_dead_discrimination": 0.021, + "mean_active_discrimination": 0.03, + "dead_dims_encode_semantics": true + } +} \ No newline at end of file diff --git a/data/exp_g3_analysis.json b/data/exp_g3_analysis.json new file mode 100644 index 0000000000000000000000000000000000000000..bba722debb85d8d0709944d25ad1ee886e3f8517 --- /dev/null +++ b/data/exp_g3_analysis.json @@ -0,0 +1,110 @@ +{ + "mean_pearson_r": 0.0153, + "mean_slope_ratio": -0.0196, + "H0_supported": false, + "H0_partial": false, + "per_file": { + "EleutherAI--pythia-1b_T1000": { + "model": "EleutherAI/pythia-1b", + "T_seq": 1000, + "ln_T": 6.908, + "n_layers": 16, + "pearson_r_H_1minusR": 0.075, + "pearson_r_H_R": -0.075, + "ols_slope": 0.1629, + "slope_vs_lnT_ratio": 0.0236, + "mae_vs_theory": 2.5022, + "H_attn_mean": 3.3132, + "H_attn_range": [ + 2.106, + 4.388 + ], + "R_max": 0.9076 + }, + "EleutherAI--pythia-70m_T500": { + "model": "EleutherAI/pythia-70m", + "T_seq": 500, + "ln_T": 6.215, + "n_layers": 6, + "pearson_r_H_1minusR": 0.6141, + "pearson_r_H_R": -0.6141, + "ols_slope": 0.5616, + "slope_vs_lnT_ratio": 0.0904, + "mae_vs_theory": 5.9782, + "H_attn_mean": 2.489, + "H_attn_range": [ + 0.674, + 3.681 + ], + "R_max": 1.2949 + }, + "gpt2-large_T200": { + "model": "gpt2-large", + "T_seq": 200, + "ln_T": 5.298, + "n_layers": 36, + "pearson_r_H_1minusR": -0.2319, + "pearson_r_H_R": 0.2319, + "ols_slope": -0.4107, + "slope_vs_lnT_ratio": -0.0775, + "mae_vs_theory": 2.3358, + "H_attn_mean": 2.256, + "H_attn_range": [ + 1.418, + 3.782 + ], + "R_max": 1.0004 + }, + "gpt2-medium_T200": { + "model": "gpt2-medium", + "T_seq": 200, + "ln_T": 5.298, + "n_layers": 24, + "pearson_r_H_1minusR": -0.152, + "pearson_r_H_R": 0.152, + "ols_slope": -0.3266, + "slope_vs_lnT_ratio": -0.0616, + "mae_vs_theory": 2.6443, + "H_attn_mean": 2.1176, + "H_attn_range": [ + 1.268, + 4.093 + ], + "R_max": 0.932 + }, + "gpt2-medium_T500": { + "model": "gpt2-medium", + "T_seq": 500, + "ln_T": 6.215, + "n_layers": 24, + "pearson_r_H_1minusR": -0.2643, + "pearson_r_H_R": 0.2643, + "ols_slope": -0.6838, + "slope_vs_lnT_ratio": -0.11, + "mae_vs_theory": 3.0876, + "H_attn_mean": 2.5431, + "H_attn_range": [ + 1.569, + 4.912 + ], + "R_max": 0.932 + }, + "gpt2-xl_T200": { + "model": "gpt2-xl", + "T_seq": 200, + "ln_T": 5.298, + "n_layers": 48, + "pearson_r_H_1minusR": 0.0509, + "pearson_r_H_R": -0.0509, + "ols_slope": 0.0943, + "slope_vs_lnT_ratio": 0.0178, + "mae_vs_theory": 1.8847, + "H_attn_mean": 2.2185, + "H_attn_range": [ + 1.427, + 4.104 + ], + "R_max": 1.0052 + } + } +} \ No newline at end of file diff --git a/data/exp_g4_theta_star_analysis.json b/data/exp_g4_theta_star_analysis.json new file mode 100644 index 0000000000000000000000000000000000000000..4a94f0951f020c2b1f662dfb58a8c2ea80dce39c --- /dev/null +++ b/data/exp_g4_theta_star_analysis.json @@ -0,0 +1,92 @@ +{ + "formula": "gamma = 1 - T_eval*sqrt(2)/theta", + "T_eval": 2000, + "n_models": 8, + "regime_A": [ + "pythia-70m", + "gemma-2-9b-it" + ], + "regime_B": [ + "Mistral-7B-v0.1", + "Meta-Llama-3-8B", + "Qwen2.5-7B" + ], + "anomalies": [ + "Llama-2-7b-hf", + "falcon-7b", + "Mistral-Nemo-Instruct-2407" + ], + "theta_star_range": [ + 10000, + 10000 + ], + "theta_star_geometric_mean": 10000.0, + "theta_star_from_formula": 2828.0, + "formula_err_regimeA_mean_pct": 6.5, + "all_models": [ + { + "name": "pythia-70m", + "theta": 10000, + "gamma_obs": 0.7476, + "gamma_pred": 0.7172, + "err_pct": 4.1, + "regime": "A (formula valid)" + }, + { + "name": "Llama-2-7b-hf", + "theta": 10000, + "gamma_obs": 0.2871, + "gamma_pred": 0.7172, + "err_pct": 149.8, + "regime": "A-ANOMALY (formula fails)" + }, + { + "name": "Mistral-7B-v0.1", + "theta": 10000, + "gamma_obs": 1.2131, + "gamma_pred": 0.7172, + "err_pct": 40.9, + "regime": "B (gamma~1, long-context)" + }, + { + "name": "gemma-2-9b-it", + "theta": 10000, + "gamma_obs": 0.6586, + "gamma_pred": 0.7172, + "err_pct": 8.9, + "regime": "A (formula valid)" + }, + { + "name": "falcon-7b", + "theta": 10000, + "gamma_obs": 0.8928, + "gamma_pred": 0.7172, + "err_pct": 19.7, + "regime": "A-ANOMALY (formula fails)" + }, + { + "name": "Meta-Llama-3-8B", + "theta": 500000, + "gamma_obs": 1.0455, + "gamma_pred": 0.9943, + "err_pct": 4.9, + "regime": "B (gamma~1, long-context)" + }, + { + "name": "Qwen2.5-7B", + "theta": 1000000, + "gamma_obs": 0.9967, + "gamma_pred": 0.9972, + "err_pct": 0.0, + "regime": "B (gamma~1, long-context)" + }, + { + "name": "Mistral-Nemo-Instruct-2407", + "theta": 1000000, + "gamma_obs": 0.5407, + "gamma_pred": 0.9972, + "err_pct": 84.4, + "regime": "A-ANOMALY (formula fails)" + } + ] +} \ No newline at end of file diff --git a/data/exp_gamma_field/EleutherAI--pythia-1.4b_gamma_field.json b/data/exp_gamma_field/EleutherAI--pythia-1.4b_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..f6d086beef7f48ac8097e2ef52d646383bb70e71 --- /dev/null +++ b/data/exp_gamma_field/EleutherAI--pythia-1.4b_gamma_field.json @@ -0,0 +1,1020 @@ +{ + "model": "EleutherAI/pythia-1.4b", + "theta": 10000, + "T_train": 2048, + "n_layers": 24, + "n_heads": 16, + "d_head": 128, + "n_prompts": 100, + "seeds": [ + 42, + 123 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "gamma_matrix": [ + [ + 1.1578705755501353, + 0.8869080388850906, + 0.7326259225540419, + 1.7568234255270307, + 0.7737628763770268, + 1.027770791312411, + 1.188452195783214, + 1.7480755880752452, + 0.8715376202538662, + 1.0193700499119966, + 0.9653162141757919, + 0.8191986466310118, + 0.6976663489293273, + 0.9218885596229991, + 1.0285592987754697, + 1.0913504907689993 + ], + [ + 0.6619121420835893, + 0.883352361372461, + 0.9081702486392946, + 0.8817911191391561, + 0.6835453773790271, + 0.856332019190857, + 0.2516771877457767, + 1.0645893681070457, + 0.26581995706907885, + 0.8410673866840657, + 0.7980002484278271, + -0.25489405996629166, + 1.3383257755627036, + 0.6538827979832438, + 0.5540736302307779, + 0.5676222306864117 + ], + [ + 1.674527478831519, + 0.7590668486568224, + 0.6669934242185699, + 0.6604324165448482, + 0.7572712400258526, + 0.8072127699758772, + 0.7465735697346123, + 0.8945163480750086, + 0.7921736645780619, + 0.5692075707301391, + 0.6290002028408747, + 0.8011338903433171, + 0.7415712392433518, + 0.6836217293228267, + 0.7872838435863282, + 0.7869783019258745 + ], + [ + 0.8531151664834982, + 0.3764460595205523, + 0.6859567419556265, + 0.5829924990807441, + 0.8771869087415287, + 0.46954012354576935, + 0.7578516461858689, + 0.6965915457265051, + 0.7132842942977466, + 0.7031545275526778, + 0.39405273005298147, + 0.6219794039722086, + 0.7094588057709719, + 0.7928518435001258, + 0.813037862544174, + 0.6755930238258767 + ], + [ + 0.8468909355883746, + 0.7973868148701114, + 0.5404281634096747, + 0.9546793011186373, + 0.9125645542099609, + 0.3068098340700925, + 1.06426886824475, + 0.7252891013047493, + 0.6626658562302794, + 0.6448580321870137, + 0.736751689911295, + 0.6739943425167195, + 0.42317315822323526, + 0.9120510310857243, + 0.4814634054342153, + 0.2912717123520083 + ], + [ + 0.8934231772400477, + 0.7083471911821391, + 0.6178733720633727, + 1.0513865762303845, + 0.5816506388382618, + 1.0502098675165952, + 0.5971313290529853, + 0.8791379724741453, + 0.7046456057020707, + 0.5481878008350556, + 0.551179625127339, + 1.3914957544753166, + 0.47748984697715796, + 0.7127975564121859, + 0.6330261778184991, + 0.9496956488266157 + ], + [ + 0.6205458254934217, + 0.9961477740726046, + 1.2928429567505675, + 1.121062797925592, + 0.7064020665747581, + 1.0868969017284875, + 0.9829220920874581, + 1.1190806952493568, + 0.9248067070733913, + 0.9674804391960732, + 0.663556361046569, + 0.9008569027076863, + 0.868352937354718, + 0.6385910432755296, + 0.2992340164355994, + 0.5542002430555188 + ], + [ + 0.43140845301299385, + 1.0748849393613427, + 0.6049310130382731, + 0.40644271818178956, + 0.853224111834578, + 0.5596731652720067, + 0.7611937940017302, + 0.6979076541930821, + 0.3322237391676036, + 0.773272934154798, + 0.6204578530083463, + 0.6737067334012345, + 0.7506869433883868, + 0.6984988490617156, + 1.1909222981740566, + 1.8557244919069582 + ], + [ + 0.756083012119903, + 0.7539311833286334, + 0.8019819558190359, + 1.6305401571100395, + 0.608270363205928, + 0.6094958130266892, + 0.7820954140940869, + 0.6264281345937356, + 1.0501485014972658, + 0.983185664178738, + 0.9681493573839137, + 1.11947279519518, + 0.983902748695303, + 0.7498435299916729, + 0.6546101836473646, + 1.640102948177106 + ], + [ + 0.8790281981270099, + 0.8717039647707567, + 0.7768908415732277, + 0.9111812979169022, + 0.9499743399033278, + 0.8792854514176403, + 1.1907950596008159, + 1.0842344394617656, + 0.6448717213075927, + 0.7598130006362595, + 0.7709416148822013, + 0.8336964859367143, + 0.8200006328665633, + 0.8448119551660931, + 0.5865009844126516, + 0.9888644434626436 + ], + [ + -0.05241901064342408, + 1.2100253778085042, + 1.3129960087961563, + 0.9519206344934726, + 0.8578815110516801, + 0.6731141277977598, + 0.9597260585788245, + 0.7697042526859117, + 0.5239664838495783, + 0.5962570844278307, + 1.0389558460691914, + 0.71736848625201, + 1.2620655186009841, + 1.162785944733063, + 1.2941325325017983, + 1.0635856503822299 + ], + [ + 0.739523273436235, + 0.9965437510047259, + 0.3226870471439754, + 0.6512718219813171, + 0.496767390245986, + 0.5938437848569355, + 0.6224258481901145, + 1.205732843602345, + 0.6575854376491866, + 1.883511413175968, + 0.6571255528177663, + 0.6717922548455902, + 0.6177218085538658, + 0.6952837335526745, + 0.7561957872149548, + 0.6807507394805915 + ], + [ + 0.3991723838901931, + 0.8009560496428203, + 1.0680774859422257, + 1.1442117538093441, + 1.4401954866836024, + 0.9192506945261206, + 0.8272868775008038, + 0.7092579316299242, + 0.7372707915401595, + 1.137913531253263, + 0.8795518926995746, + 0.5108457696183433, + 1.108931667785543, + 1.2684434345071716, + 0.7061845740562657, + 0.4391896586704156 + ], + [ + 0.7492162113746534, + 0.6866999199414506, + 0.7759647127985789, + 0.4995321952997332, + NaN, + 0.6334950361719205, + 1.0429819934662905, + 1.1135536897836926, + 0.8613945456974239, + 2.2160171797340684, + 1.271240454152724, + 0.8285145161624956, + 0.7527142269198872, + 0.8180403186132406, + 1.0226709832872274, + 0.6363004657847908 + ], + [ + 0.5394992717212015, + 1.0878184678140423, + 0.6096053276765774, + 2.4203902835840094, + 0.7982269602337413, + 0.8288119503876228, + 0.3945644833687705, + 0.8001136408497334, + 1.5625955675025887, + 1.2651218311656685, + 1.1038042849239278, + 0.5460236315036509, + 0.8040443717306823, + 1.0955002335310948, + 1.392567098798463, + 0.5904281727128783 + ], + [ + 0.7967650707130752, + 0.6563748138700621, + 0.8051661676937927, + 0.6899304634690816, + 0.9435390817272916, + 1.108257359583726, + 0.9161735892028474, + 0.6070356252837706, + 1.002218446910503, + 0.4817594191240528, + 2.0524720045524676, + 0.495700549389223, + 0.7653839912263524, + 0.6774698903583868, + 0.5888062287496618, + 0.5577500865399752 + ], + [ + 0.5119018646684848, + 0.51557857429562, + 0.6396286571013124, + 0.5704675335894112, + 0.47148990092213794, + 0.5398596301795913, + 1.1022025052581612, + 1.0622462002911712, + 1.6456192159420966, + 2.2193666164103862, + 0.4051146576113961, + 1.4245384371044199, + 1.143283144159944, + 0.34543129064441475, + 1.168532527594249, + 0.7174768939397109 + ], + [ + 0.7353351903972097, + 0.806338004743141, + NaN, + 1.9415043916955064, + 1.224995052551576, + 0.5409258210392847, + 0.6271976050247828, + 0.6791416370266732, + 0.6378281279305216, + 1.2003723598933478, + 0.8386345241082701, + 2.22671365190291, + 1.0117932274819157, + 0.6990753036646199, + 0.8403830871420771, + 0.40083982423344916 + ], + [ + 0.4769916071973614, + 0.8854823122237984, + 0.9349862056112912, + 1.148543203789438, + 0.38612009462722174, + 1.3201865056256084, + 0.5843236325448012, + 1.132616131922907, + 0.6739988234452708, + 0.7097365906211182, + 0.9148628812238053, + 0.7759109996093891, + 0.968782897946746, + 0.3631561016646753, + 1.267801637758006, + 0.41274816319105573 + ], + [ + 1.5278747630585976, + 0.8566722940393973, + 0.8201799834960329, + 0.43947189517911905, + 1.3500674833522632, + 0.7080657039335528, + 0.6024955674421296, + 0.8442085958685488, + 0.6838379216212527, + 1.0056405745701849, + 0.4605738886858128, + 0.7780568917414783, + 0.4259371025098511, + 1.6651951270985301, + 1.0221401787153197, + 0.7432112267140916 + ], + [ + 0.7759557839781828, + 0.737097659327614, + 1.2169413197386445, + 0.8208416431584058, + 2.138239573671578, + 1.929080771682426, + 0.8041509040645939, + 0.5381608300745534, + 0.5673947274417, + 1.2228157132604667, + 0.7048837935632173, + 1.115199609182677, + 0.7253948495822524, + 0.7819768261122423, + 0.6296491680844711, + 0.7817522895228818 + ], + [ + 0.8645150570127204, + 1.1599832012311022, + 0.6531787172944967, + 1.4404636633581216, + 1.7475445457203558, + 0.5223477217846869, + 1.901050935194978, + 1.0047781831090126, + 0.3576365374986829, + 1.0935818748137236, + 0.8870733105163895, + 2.0098559029009024, + 1.8252018820620541, + 0.011332084773966293, + 2.301767916910281, + 0.9371447524729827 + ], + [ + 0.81058339983165, + 2.0135375908106194, + 1.5013374327456124, + 1.7622648999886907, + 1.0124229330524568, + 1.3354818316095969, + 0.7494991623030244, + 2.130252691651698, + 2.588531049483412, + 3.3388023092965446, + 1.7218389801833536, + 1.0720161888496287, + 4.18773983086798, + 0.9285565461599831, + 2.3562026475102673, + 1.7831938872873703 + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ] + ], + "r2_matrix": [ + [ + 0.9773679868596847, + 0.8584411178100798, + 0.8638907089132817, + 0.9356772912210161, + 0.7935132184470719, + 0.9527357727470953, + 0.4720971364316928, + 0.9610789122232798, + 0.7089571656515286, + 0.9671727375328242, + 0.8808794846915471, + 0.7442959057170688, + 0.6414271496912936, + 0.866415437740456, + 0.6962146900292341, + 0.9720872010364279 + ], + [ + 0.6899195684341277, + 0.8815217399337465, + 0.9585631974799882, + 0.6794092962219103, + 0.5855500805558609, + 0.3593469325275186, + 0.06519022903384664, + 0.8630014878131347, + 0.07171051883225166, + 0.849898856399917, + 0.31362271857879054, + 0.09170368408219343, + 0.6811989136079954, + 0.5332707360882575, + 0.40320661816834213, + 0.4203520630772365 + ], + [ + 0.9357283181274062, + 0.759831139008855, + 0.8475032261249447, + 0.9592405220908535, + 0.6855809073066894, + 0.48816836716594847, + 0.7985274527499966, + 0.9588184997179114, + 0.8796887713412127, + 0.7947673675823983, + 0.8928440617332164, + 0.9044191366660912, + 0.8001014867732741, + 0.8221495956471563, + 0.7352616663839634, + 0.3222036415120052 + ], + [ + 0.8847983400207401, + 0.3100371237800168, + 0.7989621668795583, + 0.4283611410935433, + 0.7788546873481583, + 0.16605734400351335, + 0.822040423802542, + 0.9043874534924313, + 0.5866346643055358, + 0.8831851572942305, + 0.29969051203011055, + 0.8306197908385744, + 0.7702329887949764, + 0.7158533158928955, + 0.9723948530039024, + 0.346362462292663 + ], + [ + 0.957393467112624, + 0.7868349663279036, + 0.9370846157169401, + 0.9614512522591618, + 0.9352921409423365, + 0.03439689766132048, + 0.4769929541032303, + 0.6250179280813726, + 0.9551914013331498, + 0.7922267181525532, + 0.8111515139109782, + 0.8671824966217636, + 0.6919014845606545, + 0.8691366344897572, + 0.10881619773690587, + 0.029130014121247494 + ], + [ + 0.2694564174073556, + 0.8730853588367662, + 0.7224135937181515, + 0.4542135707019822, + 0.7259596220619485, + 0.9716769364399195, + 0.2929265869652211, + 0.7476519792202089, + 0.9582549155831156, + 0.12023143916180146, + 0.9455486611558914, + 0.7734723667792512, + 0.4577148557452444, + 0.9270304005781294, + 0.562177062186322, + 0.5155058557472058 + ], + [ + 0.4807002951646101, + 0.5289280263421466, + 0.361290792389924, + 0.5176546216107282, + 0.47302455185679604, + 0.9729542445690654, + 0.9921776532944048, + 0.30561982436919966, + 0.25323779522971457, + 0.7405626625422765, + 0.1315398507932921, + 0.684497914034208, + 0.7688578529124162, + 0.7772252389220069, + 0.1309483386518322, + 0.1136439217861408 + ], + [ + 0.1419471125673898, + 0.9329986964575478, + 0.8439307075481994, + 0.718336964067849, + 0.9371699793100429, + 0.06826197637092479, + 0.9788961557469518, + 0.3428028508019697, + 0.0916187140444148, + 0.36676216091485303, + 0.9566532161303262, + 0.8441715949292024, + 0.7201213910687314, + 0.8791588436737106, + 0.2484449850782855, + 0.9046959228498097 + ], + [ + 0.9363050165869371, + 0.8440742660430347, + 0.1746208325292784, + 0.9626043830448217, + 0.14692759992546756, + 0.9123374629495572, + 0.9913846757764097, + 0.9014710305552202, + 0.9433007569771686, + 0.374800756212157, + 0.9416588697101234, + 0.26916491440798096, + 0.4745198013983357, + 0.8871728323853456, + 0.10113850009063008, + 0.4944444326762897 + ], + [ + 0.9052696634342764, + 0.7642385024647486, + 0.9240521827469665, + 0.5432048026456514, + 0.5931239846153616, + 0.907908841675195, + 0.727064022552516, + 0.4387292581830351, + 0.8796182346905058, + 0.9410831448643274, + 0.967197896233256, + 0.7960143152058471, + 0.9691944684332475, + 0.9974882181251861, + 0.770699833120387, + 0.8644784138752455 + ], + [ + 0.03875359832067771, + 0.3022337423650421, + 0.3369418416845571, + 0.9917741453205103, + 0.9359968306431764, + 0.9471099154616546, + 0.9141807614511946, + 0.46805884664418484, + 0.9281731091829886, + 0.9299071974701135, + 0.9543834038667702, + 0.8883507362875234, + 0.6990149728327646, + 0.9441172526874569, + 0.5486174554564299, + 0.9491254386728807 + ], + [ + 0.9598462676928613, + 0.7785620627683267, + 0.09193149817209878, + 0.8879601172749554, + 0.09840384814625591, + 0.8528042806969938, + 0.8565791099244209, + 0.4253192148243419, + 0.9791567040088298, + 0.5014897520529046, + 0.902757531792761, + 0.9546315918206828, + 0.9031977534091791, + 0.9337023627472798, + 0.9567630523866336, + 0.8723026985561301 + ], + [ + 0.5039913729542058, + 0.7495061217783586, + 0.7078304403206717, + 0.16887325941649023, + 0.6876170704689253, + 0.311389964262064, + 0.8684931715642753, + 0.9348377026678385, + 0.21076918086196228, + 0.8991371665404775, + 0.9823558891208274, + 0.7726147755360027, + 0.27455538593247675, + 0.9556734506850217, + 0.9458475585522902, + 0.5618287515030428 + ], + [ + 0.9369983184206073, + 0.8279359777933312, + 0.8706753034800976, + 0.20959354623656323, + 0.0, + 0.6163795445246109, + 0.7799921367067197, + 0.9129932558735037, + 0.9678370325483555, + 0.5906440396421514, + 0.7780426139917794, + 0.8135995376812503, + 0.4832109069940983, + 0.9503204492611244, + 0.7393159945133789, + 0.8863954625099694 + ], + [ + 0.9509030570039909, + 0.9020149810523833, + 0.8944904293312812, + 0.5797758609624702, + 0.8820786196662341, + 0.9479827109982604, + 0.7995376338719221, + 0.8779137431221069, + 0.6521518156126196, + 0.9187485644223256, + 0.6660793854165978, + 0.8845385703823172, + 0.7798740513804534, + 0.7889091064231554, + 0.770514487852954, + 0.7353417726587483 + ], + [ + 0.8570914685279953, + 0.8563534125154693, + 0.6713518013300551, + 0.8424341995864919, + 0.9098517422420231, + 0.4476809937690259, + 0.9519198360316451, + 0.882138079563687, + 0.6367668257152845, + 0.8638644043233581, + 0.5501527635615955, + 0.7344905873501042, + 0.2700425614772438, + 0.9601944001759699, + 0.9125443000136122, + 0.8413871578457219 + ], + [ + 0.9033093882706102, + 0.7544635126817979, + 0.9635886688130304, + 0.9094508006677501, + 0.9542660389171647, + 0.8788156453246374, + 0.9568803154705557, + 0.6233223967671919, + 0.4984561471677127, + 0.8803141588474971, + 0.43848459591126543, + 0.8244648004830886, + 0.2692787797304106, + 0.5536475347951302, + 0.7520131049535892, + 0.8219650209964624 + ], + [ + 0.894167412419102, + 0.8904008256306505, + 0.0, + 0.7922499390782095, + 0.7448883037165184, + 0.8455291109447175, + 0.7859328722359297, + 0.8581531258026422, + 0.9028070916411471, + 0.689650486359899, + 0.9025951954796047, + 0.6052144929759065, + 0.7490213389680973, + 0.864672692365134, + 0.5818963758725573, + 0.8485915354199023 + ], + [ + 0.8456607089305056, + 0.4422656674475812, + 0.9251989415785142, + 0.4496596641841043, + 0.6833589440563748, + 0.8205460145339079, + 0.8643424032882381, + 0.6727858324731815, + 0.7936175318854913, + 0.8342985805616171, + 0.45714224145915094, + 0.9138323309455658, + 0.7506169390380524, + 0.39540138214380016, + 0.7051044666844359, + 0.4906912568547086 + ], + [ + 0.739593366639754, + 0.6074658504314255, + 0.9724316938032742, + 0.7661108836580311, + 0.5626845522913609, + 0.9567078330666281, + 0.8622868618999111, + 0.7602747407870066, + 0.7444398767719012, + 0.8989006192281664, + 0.8682746111729904, + 0.864774068019408, + 0.7181198869035537, + 0.484338511813317, + 0.3917785173277093, + 0.44263503480408084 + ], + [ + 0.846861903381786, + 0.8163687868998445, + 0.9518471693261111, + 0.8920442614540242, + 0.8169727637952598, + 0.8773218440088184, + 0.8539525229286083, + 0.30053140662024946, + 0.8425996248128541, + 0.6596364546331163, + 0.8858389764482074, + 0.6281665490840767, + 0.802990900493462, + 0.841844331648512, + 0.8673537209504006, + 0.8634036004970463 + ], + [ + 0.8244981288670352, + 0.8587721808344058, + 0.7513741572350361, + 0.8628682307761688, + 0.8740729796301934, + 0.887588415823501, + 0.9121433334444385, + 0.7643440483933089, + 0.4728791461734656, + 0.8975989855832018, + 0.6648416843208729, + 0.7547629652575658, + 0.9639956008065864, + 0.00017739536526295563, + 0.9238924518987256, + 0.33251414507248767 + ], + [ + 0.9096487073203757, + 0.7934193734822155, + 0.6399221005066855, + 0.6705408785459371, + 0.8991033715765749, + 0.7503785228399129, + 0.6833746947875334, + 0.8791808779676732, + 0.8994188893443498, + 0.835680535428643, + 0.874223478063916, + 0.48375505411158604, + 0.6842640201589338, + 0.9740118465284484, + 0.8788914447778934, + 0.5231681009512551 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + ], + "summary": { + "gamma_mean": 0.9066859893753302, + "gamma_std": 0.4627190613451061, + "gamma_min": -0.25489405996629166, + "gamma_max": 4.18773983086798, + "gamma_range": 4.442633890834272, + "gamma_global_fit": 0.6900401428012509, + "r2_global": 0.973692105664348, + "gamma_mixture_pred": 0.9066859893753302 + }, + "outlier_heads_high": [ + [ + 7, + 15, + 1.8557244919069582 + ], + [ + 11, + 9, + 1.883511413175968 + ], + [ + 13, + 9, + 2.2160171797340684 + ], + [ + 14, + 3, + 2.4203902835840094 + ], + [ + 15, + 10, + 2.0524720045524676 + ], + [ + 16, + 9, + 2.2193666164103862 + ], + [ + 17, + 3, + 1.9415043916955064 + ], + [ + 17, + 11, + 2.22671365190291 + ], + [ + 20, + 4, + 2.138239573671578 + ], + [ + 20, + 5, + 1.929080771682426 + ], + [ + 21, + 6, + 1.901050935194978 + ], + [ + 21, + 11, + 2.0098559029009024 + ], + [ + 21, + 14, + 2.301767916910281 + ], + [ + 22, + 1, + 2.0135375908106194 + ], + [ + 22, + 7, + 2.130252691651698 + ], + [ + 22, + 8, + 2.588531049483412 + ], + [ + 22, + 9, + 3.3388023092965446 + ], + [ + 22, + 12, + 4.18773983086798 + ], + [ + 22, + 14, + 2.3562026475102673 + ] + ], + "outlier_heads_low": [ + [ + 1, + 11, + -0.25489405996629166 + ], + [ + 10, + 0, + -0.05241901064342408 + ] + ] +} \ No newline at end of file diff --git a/data/exp_gamma_field/EleutherAI--pythia-1b_gamma_field.json b/data/exp_gamma_field/EleutherAI--pythia-1b_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..7259d91d19fc4c99683ce71f5f7466f148281345 --- /dev/null +++ b/data/exp_gamma_field/EleutherAI--pythia-1b_gamma_field.json @@ -0,0 +1,391 @@ +{ + "model": "EleutherAI/pythia-1b", + "theta": 10000, + "T_train": 2048, + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "n_prompts": 60, + "seeds": [ + 42, + 123 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "gamma_matrix": [ + [ + 0.8396112180516474, + 0.9914095037789742, + 1.2416689332061102, + 1.0804016840389459, + 2.203744521319629, + 1.4746688428830486, + 1.1026709741353247, + 1.002995815732713 + ], + [ + 0.07687109379316528, + 1.529635180045567, + 0.5831331955549788, + 2.771663542760671, + 1.313027619284348, + 1.4422100878884128, + 1.1648376759133048, + 1.0804063773134147 + ], + [ + 0.8553281074214615, + 1.4710431662719157, + 0.9381328974929382, + 1.315087940091258, + 1.14866408115732, + 0.585382940745927, + 0.8199952224696406, + 1.0663723257670654 + ], + [ + 1.8101410343543036, + 1.126507826107865, + 0.5621118057940716, + 0.7362932900578493, + 0.7532942418052956, + NaN, + 0.5359664140957514, + 0.8186297547762317 + ], + [ + 0.9630318747509097, + 1.1561722707972302, + 0.8125462929986226, + 0.6766492950291022, + 0.6151560357369132, + 0.8678998854312073, + 0.8521575330683555, + 0.603631496764009 + ], + [ + 0.5191141537427254, + 0.884227659909035, + 0.5923766743832868, + 0.928404202465268, + 0.9147902164573699, + 0.5729548557364857, + 0.920444887952919, + 0.523575949475416 + ], + [ + 1.1064122690923346, + 0.9086783217129951, + 0.8166337833709647, + 0.5947838382393198, + 0.8731120536926595, + 0.9767716930284652, + 0.9370350522283324, + 0.6919144310045783 + ], + [ + 0.6691029444451231, + 0.6205926029894723, + 0.8360148923303251, + 0.8588159670496819, + 1.2632009645958397, + 1.4491845666569996, + 1.2153333682391885, + 1.1880594206699382 + ], + [ + 0.9265351760846104, + 0.8595884369338112, + 0.9751132036837779, + 0.9084968794142233, + 1.1097610078506142, + 1.00107851654304, + 1.0109014681132495, + 1.2067776431779051 + ], + [ + 0.8767314834339544, + 0.7558729242460741, + 1.001250788030399, + 0.8963188099194975, + 0.7443435269408117, + 0.7398102433729618, + 1.088042707488966, + 0.9217848241821661 + ], + [ + 0.8809387897259895, + 1.5494629132916073, + 0.9111819059128659, + 0.969936961366972, + 0.936285198218925, + 0.5195418574900117, + 0.7336271225219896, + 1.0120427539125905 + ], + [ + 0.9122350439075307, + 0.7827972407305469, + 0.6343020326510451, + 0.7800374014382553, + 0.8883874023584357, + 0.6854077337780102, + 0.7127771706267649, + 0.6565801853947911 + ], + [ + 0.6890077735865242, + 0.9714802739441339, + 0.45521776047098955, + 0.5470802893832037, + 0.9024164389143299, + 0.8377769347199506, + 0.7212189685918424, + 0.7765806040354803 + ], + [ + 1.1549000546979742, + 0.7676330425869643, + 0.5459691673329192, + 0.8597185320284861, + 1.3589542536355885, + 0.7478785875104619, + 0.7325495583511101, + 0.6259703590547835 + ], + [ + 1.3953482388254563, + 0.8069061073295272, + 1.2930708904189399, + 1.1588332685395801, + 0.8715664415033831, + 1.2224643555040287, + 0.7234147111921838, + 0.9626354953921575 + ], + [ + 1.19234583384983, + 0.9273209849786251, + 1.2483316293853761, + 0.8562698834639803, + 0.7655228936664625, + 1.0581076179122268, + 0.8949658879582604, + 1.2324387946001427 + ] + ], + "r2_matrix": [ + [ + 0.8831451615370409, + 0.9549849286846273, + 0.9630137777881698, + 0.8807494913679469, + 0.9866999724517822, + 0.9861090427739921, + 0.9799938369630977, + 0.9374956279224039 + ], + [ + 0.004793225759246678, + 0.9159731395186271, + 0.8003696481120883, + 0.7555578544852265, + 0.352698181084545, + 0.819183134220076, + 0.8862245803992508, + 0.9054225354625551 + ], + [ + 0.8604848088339963, + 0.8970121961990759, + 0.6603519678006495, + 0.8189969219722247, + 0.9768504103855251, + 0.35901178922730637, + 0.931241568920192, + 0.9799142576925339 + ], + [ + 0.9181719171020266, + 0.977059970492666, + 0.32522886185182653, + 0.8204487526671624, + 0.8793478665422716, + 0.0, + 0.37744724634613513, + 0.6985736651019185 + ], + [ + 0.9201859766982319, + 0.4586860491735878, + 0.8148395564239657, + 0.8232060648845021, + 0.5681295220024563, + 0.9502661686375705, + 0.8451049818501382, + 0.1667016711190169 + ], + [ + 0.4032005397082239, + 0.7134906803669654, + 0.7076833742116984, + 0.24067903888208342, + 0.9619511949629312, + 0.126872997901189, + 0.950184988933867, + 0.519758820105462 + ], + [ + 0.23288935935186073, + 0.9645129901703161, + 0.6963028516054642, + 0.15967298015810183, + 0.40460282781647827, + 0.2567259575352052, + 0.665799672504472, + 0.6557448784240586 + ], + [ + 0.8843547039946529, + 0.8447929212354321, + 0.8714658163161091, + 0.1357048567092496, + 0.8308387132591101, + 0.20099759262405037, + 0.20038701232630896, + 0.548995431382633 + ], + [ + 0.8391308752688642, + 0.6813314879636447, + 0.8145185563537689, + 0.37278930589545767, + 0.35148710044049947, + 0.7880079751677556, + 0.6912873021850844, + 0.36488562912443057 + ], + [ + 0.7740123724749508, + 0.8873794002063756, + 0.28749450100460394, + 0.4742275027128813, + 0.541219145594414, + 0.7974300486746629, + 0.33007809431232227, + 0.8648564561762206 + ], + [ + 0.9827150747884983, + 0.20500408791630143, + 0.39935377511730574, + 0.38717144193214503, + 0.3729592999795994, + 0.9482021288756908, + 0.954181400712733, + 0.5192765334079961 + ], + [ + 0.7292418551049387, + 0.883010709864236, + 0.7299388535681016, + 0.5879627695374501, + 0.856032146695642, + 0.9195262598595575, + 0.9543950845847243, + 0.9589162211621126 + ], + [ + 0.9497585714349279, + 0.4391206312305339, + 0.8157297205763074, + 0.8196054332135931, + 0.40999702652193437, + 0.4444951698303582, + 0.8956040512652186, + 0.8481837727376205 + ], + [ + 0.344233287191253, + 0.5078094024836055, + 0.9372386467079528, + 0.9898797020590342, + 0.7584582526299921, + 0.5912250741803053, + 0.9553944815825159, + 0.5834242812183406 + ], + [ + 0.6116819390167232, + 0.7631751653078428, + 0.9568355266869597, + 0.8502526775469151, + 0.7241160976401524, + 0.7106868347264073, + 0.9565111310236821, + 0.5636891553184115 + ], + [ + 0.6228097571292635, + 0.753881629713419, + 0.8604108947876953, + 0.996779768475402, + 0.8966294526913872, + 0.6388358723884096, + 0.899854781055552, + 0.6739792421302004 + ] + ], + "summary": { + "gamma_mean": 0.9428545807422167, + "gamma_std": 0.33149603436752584, + "gamma_min": 0.07687109379316528, + "gamma_max": 2.771663542760671, + "gamma_range": 2.6947924489675055, + "gamma_global_fit": 0.7530236101524865, + "r2_global": 0.8266548452451481, + "gamma_mixture_pred": 0.9428545807422167 + }, + "outlier_heads_high": [ + [ + 0, + 4, + 2.203744521319629 + ], + [ + 1, + 3, + 2.771663542760671 + ], + [ + 3, + 0, + 1.8101410343543036 + ] + ], + "outlier_heads_low": [ + [ + 1, + 0, + 0.07687109379316528 + ] + ] +} \ No newline at end of file diff --git a/data/exp_gamma_field/EleutherAI--pythia-1b_step10000_full_gamma_field.json b/data/exp_gamma_field/EleutherAI--pythia-1b_step10000_full_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..dc874113a4de13b5ed4f1cb4b2fe2199319a771b --- /dev/null +++ b/data/exp_gamma_field/EleutherAI--pythia-1b_step10000_full_gamma_field.json @@ -0,0 +1,401 @@ +{ + "model": "EleutherAI/pythia-1b", + "theta": 10000, + "T_train": 2048, + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "n_prompts": 100, + "seeds": [ + 42, + 123 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "gamma_matrix": [ + [ + 0.8334493923117795, + 0.9759701214731358, + 1.1632283630755795, + 0.9574797727465159, + 2.062514497116373, + 1.224591586437351, + 0.9449911719600551, + 1.132995686100646 + ], + [ + 0.7400286852334905, + 1.7570714735908242, + 0.8475667920576375, + 1.711126533320611, + 0.8122994549235109, + 1.4166823047252324, + 1.1989267296462087, + 1.0986135411930813 + ], + [ + 0.5055877004019274, + 1.5785416463365314, + 0.9818610979650481, + 1.7970577894960882, + 0.8458934706875758, + 0.31531676987684215, + 0.6399210967283387, + 1.1223204202097332 + ], + [ + 1.539913966105793, + 1.1684454728448355, + 0.5459497937671179, + 0.6033082723981846, + 0.7432296852804611, + -0.8098301307498013, + 0.3409744161709651, + 0.37090780403708795 + ], + [ + 0.8666159843815948, + 0.5947504148651915, + 0.7701248254268944, + 0.6089614949757122, + 0.6952046229419875, + 0.8108707905576551, + 0.9359513757067398, + 0.3719398342982479 + ], + [ + 0.9938871603368219, + 0.8101126847405705, + 0.9145906681321125, + 0.22274418218737663, + 0.8920659178368745, + 0.263346124955582, + 0.872597762095922, + 0.7905943381096424 + ], + [ + 0.3612558861914358, + 0.8549902515342732, + 1.01173440039235, + -0.07598452803289066, + 0.8051020112457846, + 0.5736123576616793, + 0.9691053895418731, + 0.7405881218418644 + ], + [ + 1.0285152486358125, + 0.7980892551004142, + 0.857109850773097, + 0.5697683146793444, + 1.5610950706052817, + 0.4776163798007907, + 0.5743900700621248, + 0.8798847967898151 + ], + [ + 0.9556160611148582, + 0.9298320415155833, + 1.0043459331739388, + 0.9427251474876736, + 0.777887489381739, + 1.0946574444020292, + 1.003695705771028, + 0.6833714613999748 + ], + [ + 1.1655492557050806, + 1.0178905424345335, + 0.6467109519273795, + 0.9627884488557011, + 0.9444424084738572, + 1.1330804867381084, + 0.6407492537928718, + 1.0957815021629052 + ], + [ + 1.0393766003336422, + 0.49544765779315886, + 0.33592992800932947, + 0.5031428675610023, + 1.01003842034714, + 0.9824244959077766, + 0.929902975839392, + 0.9785160295647153 + ], + [ + 0.6984803994530259, + 1.0635867784699993, + 0.9765100992039869, + 0.8986708268521272, + 1.1226596493710095, + 1.1394285931591683, + 0.9855851040397622, + 1.1358227761514306 + ], + [ + 1.1042626511936788, + 0.58204213687487, + 0.9755412991133396, + 1.0501838004445148, + 0.6015738177487793, + 0.650621802433272, + 0.9246644733230025, + 1.2750038003627067 + ], + [ + 0.4178539495329005, + 0.7663083328972471, + 1.0929941057662178, + 1.1749564399994732, + 1.0958203670787565, + 0.7580869451754828, + 0.7799802124372615, + 0.996469050423769 + ], + [ + 0.6457559876839348, + 1.0753984341464384, + 1.0303157564006302, + 0.9309970322139111, + 0.8351424960595374, + 1.1855613097037587, + 0.7828316162166423, + 0.737440815301528 + ], + [ + 0.9955096053787933, + 0.9879594077245235, + 0.9829932016733423, + 1.0344261598518252, + 0.8163788007552125, + 0.8854152136226414, + 1.1682047642543778, + 1.1105636890587967 + ] + ], + "r2_matrix": [ + [ + 0.8104733576044064, + 0.9557062436197553, + 0.9097383815318889, + 0.7289865509135951, + 0.9345746663182402, + 0.9080283169080503, + 0.9908553046922304, + 0.9752581408977813 + ], + [ + 0.42598130656175426, + 0.846350176285908, + 0.9656162430464127, + 0.826877240322729, + 0.5039350420890693, + 0.9412583673689795, + 0.5985450511365922, + 0.8500659128681254 + ], + [ + 0.7121511889997132, + 0.6974407408373668, + 0.8632179818239787, + 0.8570705427375334, + 0.9567372825833517, + 0.10205455059675594, + 0.6499419386974303, + 0.9756053512795303 + ], + [ + 0.8719855824249625, + 0.9616095433915376, + 0.4264659954619908, + 0.629025165555767, + 0.9078492108159268, + 0.18007094644746247, + 0.3730108135563738, + 0.20336959378212605 + ], + [ + 0.9553021968390396, + 0.7140290568589636, + 0.9518154119326709, + 0.8708224008369811, + 0.830923365422963, + 0.9481204608813326, + 0.9724159640293285, + 0.24620111874936235 + ], + [ + 0.9559820944162766, + 0.8884385835177607, + 0.9540371406179253, + 0.4999413124240496, + 0.9623112762007654, + 0.3447105534499375, + 0.956112209980354, + 0.9512925460565271 + ], + [ + 0.6925847392517106, + 0.9834688897097849, + 0.9738033293416797, + 0.00768637227149116, + 0.8799014280848267, + 0.660912057728657, + 0.975834556254771, + 0.945852954298154 + ], + [ + 0.9881092150435884, + 0.9106238666093984, + 0.9037935321932701, + 0.7410833918346534, + 0.7933226259059285, + 0.7651070401576875, + 0.868308786906469, + 0.9333314475689664 + ], + [ + 0.9299136292872435, + 0.9591846936783452, + 0.9943838729345983, + 0.9760473726042662, + 0.8652023552631141, + 0.9789201714833098, + 0.9666963705462, + 0.8680437463170855 + ], + [ + 0.9709390565580528, + 0.9355711629193759, + 0.7545546999227073, + 0.9575033468418663, + 0.9714109660244252, + 0.9633722529342329, + 0.7888905829405781, + 0.937646298783392 + ], + [ + 0.9635372537903019, + 0.5906014166880726, + 0.45768564820003765, + 0.708906613072885, + 0.9905825950256325, + 0.9554917114117554, + 0.990499933303469, + 0.9608380843831914 + ], + [ + 0.9480594529575821, + 0.985122480784389, + 0.954651310815916, + 0.9271076514861413, + 0.9558279148371532, + 0.9757642045805685, + 0.8988985239803765, + 0.9369187420327386 + ], + [ + 0.9666432928798828, + 0.7133617424940295, + 0.9909643097646822, + 0.969665662448983, + 0.8041867961580967, + 0.6575767014787801, + 0.9909607545514857, + 0.9585625209099411 + ], + [ + 0.6247125606453287, + 0.9259097340812333, + 0.9945177551244873, + 0.9870398356217964, + 0.8333845989971624, + 0.9599851450948831, + 0.9756899840094918, + 0.988596699772356 + ], + [ + 0.7280548658120971, + 0.8640694939666873, + 0.9894386095291136, + 0.8861049345917338, + 0.8906678702537246, + 0.8797662524202383, + 0.9891997794579176, + 0.9118918734052537 + ], + [ + 0.9026645764825049, + 0.9745183934906425, + 0.9307054375211878, + 0.9589651284985476, + 0.9336450916590456, + 0.8514274684654717, + 0.9986087063480874, + 0.8712597519270038 + ] + ], + "summary": { + "gamma_mean": 0.8852661816178581, + "gamma_std": 0.352252002097594, + "gamma_min": -0.8098301307498013, + "gamma_max": 2.062514497116373, + "gamma_range": 2.872344627866174, + "gamma_global_fit": 0.8758557739227039, + "r2_global": 0.9808999901754948, + "gamma_mixture_pred": 0.8852661816178581 + }, + "outlier_heads_high": [ + [ + 0, + 4, + 2.062514497116373 + ], + [ + 1, + 1, + 1.7570714735908242 + ], + [ + 1, + 3, + 1.711126533320611 + ], + [ + 2, + 3, + 1.7970577894960882 + ] + ], + "outlier_heads_low": [ + [ + 3, + 5, + -0.8098301307498013 + ], + [ + 6, + 3, + -0.07598452803289066 + ] + ] +} \ No newline at end of file diff --git a/data/exp_gamma_field/EleutherAI--pythia-1b_step1000_full_gamma_field.json b/data/exp_gamma_field/EleutherAI--pythia-1b_step1000_full_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..7b1733ce4778707ac64ea732cc86a4300ecb45f8 --- /dev/null +++ b/data/exp_gamma_field/EleutherAI--pythia-1b_step1000_full_gamma_field.json @@ -0,0 +1,406 @@ +{ + "model": "EleutherAI/pythia-1b", + "theta": 10000, + "T_train": 2048, + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "n_prompts": 100, + "seeds": [ + 42, + 123 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "gamma_matrix": [ + [ + 0.7643217864837205, + 0.7583008846384591, + 0.7440369246158322, + 0.7524597003444957, + 0.7517980784758727, + 0.7504581341271065, + 0.7630526404229151, + 0.7421691585583287 + ], + [ + 1.0654080146813525, + 1.3181580601220737, + 1.3638758633812063, + 0.6387953086271347, + 0.804747677755539, + 1.1876954315272807, + 0.7926468816975677, + 0.753642607297473 + ], + [ + 1.1112756096430214, + 1.155915693960715, + 1.035566239116404, + 0.9266315804086012, + 1.040197820909895, + 1.0337761960791825, + 0.656301737112093, + 0.8300943378550543 + ], + [ + 1.05992634683704, + 1.3075014028296486, + 0.7483051181560331, + 0.9382998038459225, + 1.085026392120318, + -0.3002222809624908, + 0.3379057820713906, + 0.27870689895048395 + ], + [ + 1.024357242324027, + 0.941949783807678, + 0.7428044802540513, + 0.7515876914632372, + 0.763558912841196, + 0.8874720935153378, + 0.8812923427813879, + 0.8824124778109881 + ], + [ + 1.1728846876489238, + 0.7567416524537609, + 1.08386026651591, + 0.935160450597258, + 0.8412041344331691, + 0.760238679969963, + 0.9610821954031936, + 0.6967095365425893 + ], + [ + 0.8011799505716544, + 1.0889836347620756, + 0.6569715904420792, + 0.46717626939427803, + 1.0809878013918133, + 0.9011446050506902, + 1.0451523745797495, + 0.8072492905844225 + ], + [ + 0.8668685034778735, + 0.7529316377355876, + 1.137102985038932, + 0.6461127354551363, + 1.0823639493030857, + 0.9214422375618202, + 0.7726608186349743, + 0.7015777732229165 + ], + [ + 0.7951662514898198, + 0.7906985470839227, + 0.748790339199247, + 1.145237288413705, + 0.7669360644025185, + 0.9709380720667952, + 0.9460730450518757, + 0.7894462272406325 + ], + [ + 0.9100631089181473, + 1.009215994247514, + 0.9122673233288214, + 0.8871721088028464, + 1.0091374307979608, + 0.99394569624633, + 0.9748563184415199, + 1.0691673422059107 + ], + [ + 0.9679472649967518, + 0.749036077201994, + 0.6438103340464959, + 0.7176981743545853, + 0.738538603953846, + 0.8951817614733945, + 0.7075996630584106, + 0.6282678340626305 + ], + [ + 0.7328273500643889, + 1.2247719771041543, + 0.9702734390252399, + 0.7889543057051213, + 1.055257077807568, + 0.893847833838322, + 1.0844008123663575, + 0.9246724137838388 + ], + [ + 0.9684168878937636, + 0.6947373762649378, + 1.0645545387755904, + 0.9113857205104324, + 0.8299823802980411, + 0.7486329583264228, + 0.8777613658947471, + 1.077585193001541 + ], + [ + 1.028871085561998, + 0.7756817319092107, + 1.0151699025703165, + 1.3930872726619155, + 0.8047470755684063, + 0.7193289533637038, + 0.9310471109375398, + 1.2834731066248868 + ], + [ + 0.5997457886063133, + 0.9651061055956538, + 1.0254643198388256, + 0.7603228778648421, + 0.9407932826056545, + 0.9198150767621136, + 0.982738210437213, + 0.8986930459216282 + ], + [ + 0.6197842625226277, + 0.9630931345250762, + 0.8891388996252048, + 0.8485866782340576, + 0.9127126323906977, + 0.751985954438536, + 1.053838418358362, + 0.6659123238295984 + ] + ], + "r2_matrix": [ + [ + 0.9838044112568194, + 0.9881458721333432, + 0.9848890299463161, + 0.9873747089146697, + 0.984836723183823, + 0.9821553850335022, + 0.9840351094030393, + 0.9838710996622485 + ], + [ + 0.6513690306757876, + 0.796677824871332, + 0.866503954639385, + 0.4243822731555913, + 0.5249876614699414, + 0.7589161197692195, + 0.6844171814971395, + 0.960905716131325 + ], + [ + 0.8956370414840497, + 0.8184191206414371, + 0.8759686043595338, + 0.6822750873742434, + 0.8581076236317363, + 0.6612106154383351, + 0.9581278650085415, + 0.962129769499204 + ], + [ + 0.976973335451917, + 0.9480496304865479, + 0.7646668368125952, + 0.8118137078401024, + 0.8931217152206431, + 0.027045119546105778, + 0.20318183911631305, + 0.14673448089685726 + ], + [ + 0.991967114921425, + 0.8989758783259625, + 0.9827808413170096, + 0.9732849730384618, + 0.8920801433427914, + 0.8939706042319054, + 0.9597757818919727, + 0.7916626267094659 + ], + [ + 0.9424243610238467, + 0.9699349396606495, + 0.9443630409280971, + 0.9517226630947134, + 0.9751057318711343, + 0.9815176609484383, + 0.8223101985974627, + 0.8038734530052085 + ], + [ + 0.9241428845780365, + 0.9808092450453116, + 0.9507788996664249, + 0.11260424988484652, + 0.9639293000055894, + 0.987376802710503, + 0.9622668120322163, + 0.9489709493661884 + ], + [ + 0.9739550822652965, + 0.9127407536012633, + 0.9740474600846996, + 0.9020085799022252, + 0.8791250091247065, + 0.9758348508736328, + 0.982975984043518, + 0.9877812685469215 + ], + [ + 0.9151029558267736, + 0.9748580064285421, + 0.987966594454909, + 0.9383678042009753, + 0.9078257022121373, + 0.9800303812534523, + 0.9814177746199615, + 0.9450468132008317 + ], + [ + 0.9743228110745434, + 0.9946346993930207, + 0.9895835530087892, + 0.951977044507458, + 0.9405855810736858, + 0.9434486556954125, + 0.9729132115408536, + 0.9711299592970725 + ], + [ + 0.9465143038199697, + 0.9734373086611868, + 0.9954625529797331, + 0.9733572835861463, + 0.9827532785015596, + 0.9471914774895432, + 0.932164986891758, + 0.9626943897804817 + ], + [ + 0.9794164909151549, + 0.950741833069014, + 0.9289183864804141, + 0.989222392526147, + 0.9424526675425535, + 0.9539999002223021, + 0.8369073477735312, + 0.9429386278596711 + ], + [ + 0.935296286446358, + 0.9718319584408968, + 0.9750520504877429, + 0.9643294356958833, + 0.9887195919525222, + 0.9880679227413561, + 0.9467390370295404, + 0.9381814552757476 + ], + [ + 0.9365456227266996, + 0.965686294276376, + 0.9646619016437528, + 0.914464542962095, + 0.7805316015486978, + 0.9774026118339252, + 0.9661772797527866, + 0.9601830866410894 + ], + [ + 0.8988203978029242, + 0.8602968732969177, + 0.9158450877515237, + 0.7481047554166087, + 0.9166537296717651, + 0.7570162882723661, + 0.9869056420155361, + 0.9220960979132643 + ], + [ + 0.7223481962143969, + 0.8530406121492781, + 0.8741984765311, + 0.9788734656604553, + 0.9787991954183525, + 0.9094134469569225, + 0.978953551972799, + 0.7933544643456063 + ] + ], + "summary": { + "gamma_mean": 0.8779249091387495, + "gamma_std": 0.2134606263763354, + "gamma_min": -0.3002222809624908, + "gamma_max": 1.3930872726619155, + "gamma_range": 1.6933095536244063, + "gamma_global_fit": 0.8512554480180818, + "r2_global": 0.9946770598872493, + "gamma_mixture_pred": 0.8779249091387495 + }, + "outlier_heads_high": [ + [ + 1, + 1, + 1.3181580601220737 + ], + [ + 1, + 2, + 1.3638758633812063 + ], + [ + 3, + 1, + 1.3075014028296486 + ], + [ + 13, + 3, + 1.3930872726619155 + ] + ], + "outlier_heads_low": [ + [ + 3, + 5, + -0.3002222809624908 + ], + [ + 3, + 6, + 0.3379057820713906 + ], + [ + 3, + 7, + 0.27870689895048395 + ] + ] +} \ No newline at end of file diff --git a/data/exp_gamma_field/EleutherAI--pythia-1b_step143000_full_gamma_field.json b/data/exp_gamma_field/EleutherAI--pythia-1b_step143000_full_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..d72ac9bd7cf828b42eb387f16f0b92385185f9ce --- /dev/null +++ b/data/exp_gamma_field/EleutherAI--pythia-1b_step143000_full_gamma_field.json @@ -0,0 +1,391 @@ +{ + "model": "EleutherAI/pythia-1b", + "theta": 10000, + "T_train": 2048, + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "n_prompts": 100, + "seeds": [ + 42, + 123 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "gamma_matrix": [ + [ + 0.8795057152438874, + 0.9844017280364825, + 1.232778753947262, + 1.0349923106364485, + 2.208751653817526, + 1.462846291453972, + 1.0970639862945217, + 1.0553766640276696 + ], + [ + -0.004405381131756958, + 1.5283291673272372, + 0.5761287607452404, + 2.7473704383596553, + 1.332884147881978, + 1.495704418436605, + 1.172173002754719, + 1.0756171590846129 + ], + [ + 0.8599733047393633, + 1.4795641321754858, + 0.9238579361303981, + 1.2956328919752516, + 1.1541814506240533, + 0.5582701220126941, + 0.812445281530606, + 1.0719977349820158 + ], + [ + 1.8168574584690014, + 1.1336457399794755, + 0.5260499680772346, + 0.7138168257850263, + 0.7527332786764239, + NaN, + 0.5215563581960428, + 0.8172014705456233 + ], + [ + 0.8864681469569827, + 1.073884833931809, + 0.7248672221294873, + 0.5721809255224343, + 0.6129629033524642, + 0.7879224477782721, + 0.7977209663259256, + 0.5153641577256519 + ], + [ + 0.4416915602884038, + 0.8324829139855804, + 0.5776002675137978, + 0.7766618455760613, + 0.8479503740575175, + 0.42529821421510766, + 0.9091331866514428, + 0.5679290797313209 + ], + [ + 1.0175690452294357, + 0.8860533839539646, + 0.7765646199248394, + 0.2759084011457845, + 0.8262101981761801, + 0.8341184016593658, + 0.8916604380502494, + 0.6243495954385491 + ], + [ + 0.704313633534824, + 0.6176458342633918, + 0.9203540551012243, + 0.7607889057633068, + 1.1711127999539752, + 1.3627221469913524, + 0.9815926307509465, + 1.1172399677143838 + ], + [ + 0.9156256804608761, + 0.8317806744260864, + 0.976180361609469, + 0.8188931396224298, + 1.0279007193673286, + 1.0068188542002128, + 0.9964484700986493, + 1.1276092928636439 + ], + [ + 0.9019753163999291, + 0.7779401070302111, + 0.8849567816872279, + 0.8139052287300755, + 0.7027327277631024, + 0.7412426526515815, + 0.9546996100155584, + 0.8729633494738922 + ], + [ + 0.8102836466801463, + 1.4767576281433887, + 0.756579307475405, + 0.8500750457078691, + 0.81162537910177, + 0.48179861804917007, + 0.6511315892691508, + 0.9667986233083746 + ], + [ + 0.8216250023190703, + 0.7267487339911808, + 0.5407559620780636, + 0.7309714056921746, + 0.7875610695029948, + 0.6176089776823874, + 0.6901681830882597, + 0.5771686801524487 + ], + [ + 0.6662791184420488, + 0.8529484630187925, + 0.40403505459610867, + 0.5497746055876268, + 0.8449624757555881, + 0.7457605914836569, + 0.7373140069740753, + 0.8301965988541858 + ], + [ + 1.069240918982515, + 0.6708193820418654, + 0.5070678206795957, + 0.8707885072161895, + 1.2370630772562192, + 0.6147152364001347, + 0.7160433558346534, + 0.5508640316819763 + ], + [ + 1.3183257880675952, + 0.7727530283370406, + 1.1355711150150387, + 1.026633303577236, + 0.8343545797054324, + 1.1469545605685523, + 0.7246435394764129, + 0.9022721494604329 + ], + [ + 1.0407085014281543, + 0.7994866594218925, + 1.1489729658984846, + 0.7747253150596083, + 0.6835372058052135, + 0.9876955349143447, + 0.885925355325502, + 1.1111390971654063 + ] + ], + "r2_matrix": [ + [ + 0.8649454576762984, + 0.9670044821521447, + 0.9551338300060987, + 0.8922124141166533, + 0.9854971782058476, + 0.9854813333670653, + 0.9839804583815712, + 0.9524314545526558 + ], + [ + 1.656471242938995e-05, + 0.9152508722831817, + 0.8299181112613985, + 0.7548934721990418, + 0.37372021206957495, + 0.8205406454419963, + 0.8773776589945181, + 0.899733295153094 + ], + [ + 0.8471867258044904, + 0.8911793817781438, + 0.670565859236715, + 0.7789140414954946, + 0.9784299179793654, + 0.3234270338247772, + 0.9176092878899004, + 0.9823031231632287 + ], + [ + 0.9213445840701017, + 0.975579686557698, + 0.30374255806900763, + 0.7997168024769072, + 0.8915021933891273, + 0.0, + 0.4314770770734997, + 0.7021956008576038 + ], + [ + 0.9106978094291088, + 0.5168499660567778, + 0.838461201416285, + 0.7622215714202, + 0.6001841591947555, + 0.9514728269022822, + 0.8522296095886457, + 0.1684084369944986 + ], + [ + 0.4227105154470491, + 0.6424111773023775, + 0.7225893081884032, + 0.21208714055845101, + 0.9365030077729317, + 0.11489544372824301, + 0.9338718486853526, + 0.5553877784355616 + ], + [ + 0.29872449141598567, + 0.9532248635353933, + 0.689159948207108, + 0.02453892972252303, + 0.3980236225026592, + 0.22561164541901702, + 0.6634451705497462, + 0.7159221739477337 + ], + [ + 0.9318944917439921, + 0.8677863796280412, + 0.9045718921636541, + 0.12310698473320547, + 0.8108859427892663, + 0.22503422270112028, + 0.20767765492313528, + 0.5220070169578801 + ], + [ + 0.8499116882966272, + 0.7039158604258713, + 0.8461534395033061, + 0.3793298992439189, + 0.35930744806058257, + 0.8218797673168354, + 0.729935425470577, + 0.3564304980401283 + ], + [ + 0.8111031231294071, + 0.9019613179377098, + 0.27962508864442037, + 0.4573857419457775, + 0.5197514192111452, + 0.8242620593995089, + 0.32881029271951767, + 0.8707494457154135 + ], + [ + 0.9629242150365351, + 0.26773401442296685, + 0.4224096764262183, + 0.4078931914334276, + 0.4046605414827035, + 0.9014543038356045, + 0.9662855687866639, + 0.5309172311431234 + ], + [ + 0.6976098283859717, + 0.8638370741586328, + 0.6978255954720742, + 0.6656518455668117, + 0.8213042738739708, + 0.9394262893904638, + 0.956016558626492, + 0.9068968409606771 + ], + [ + 0.9238659930650726, + 0.4268916400145897, + 0.8211330461510744, + 0.8348383688147157, + 0.4166628569871327, + 0.5024546378879243, + 0.9015178820749369, + 0.9137211938419866 + ], + [ + 0.3147448243932194, + 0.5371221066019061, + 0.9804584088487549, + 0.9852552723981898, + 0.6620343507168126, + 0.5237346158010243, + 0.9457949698431312, + 0.6013837805678253 + ], + [ + 0.5793772257308513, + 0.6979803738826041, + 0.867214159573946, + 0.7610295707912886, + 0.6585317138891358, + 0.6482571257850535, + 0.9827756840769488, + 0.5360090396182322 + ], + [ + 0.5120005110853041, + 0.6864114485009468, + 0.7712733210288949, + 0.9904589354150599, + 0.8633653664091914, + 0.590940525168925, + 0.9562549508836014, + 0.5865741814791736 + ] + ], + "summary": { + "gamma_mean": 0.8917049498967125, + "gamma_std": 0.33802218644873705, + "gamma_min": -0.004405381131756958, + "gamma_max": 2.7473704383596553, + "gamma_range": 2.7517758194914124, + "gamma_global_fit": 0.7176857021389805, + "r2_global": 0.8267374615071829, + "gamma_mixture_pred": 0.8917049498967127 + }, + "outlier_heads_high": [ + [ + 0, + 4, + 2.208751653817526 + ], + [ + 1, + 3, + 2.7473704383596553 + ], + [ + 3, + 0, + 1.8168574584690014 + ] + ], + "outlier_heads_low": [ + [ + 1, + 0, + -0.004405381131756958 + ] + ] +} \ No newline at end of file diff --git a/data/exp_gamma_field/EleutherAI--pythia-1b_step1_full_gamma_field.json b/data/exp_gamma_field/EleutherAI--pythia-1b_step1_full_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..76416ce6427a0d2332e445992924f99c4372f15d --- /dev/null +++ b/data/exp_gamma_field/EleutherAI--pythia-1b_step1_full_gamma_field.json @@ -0,0 +1,391 @@ +{ + "model": "EleutherAI/pythia-1b", + "theta": 10000, + "T_train": 2048, + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "n_prompts": 100, + "seeds": [ + 42, + 123 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "gamma_matrix": [ + [ + 0.7608762831713559, + 0.7652614812440092, + 0.7484463029442969, + 0.7545435322668619, + 0.7594408749990407, + 0.7558807192271272, + 0.7626375091295992, + 0.7528338901497326 + ], + [ + 0.7754259688269534, + 0.7631360570251405, + 0.7418002440329388, + 0.7587604292601725, + 0.7576795227674133, + 0.7552327153720104, + 0.7582124304106523, + 0.7479828974070954 + ], + [ + 0.7653127806151757, + 0.7461451310788445, + 0.7696001695453099, + 0.7461037415446972, + 0.7591630737041831, + 0.7539587613487975, + 0.7538625960193533, + 0.7513070133312397 + ], + [ + 0.7563432499805288, + 0.7761107066266207, + 0.7442838840213764, + 0.7521371743450418, + 0.7505637393257434, + 0.7695447841403256, + 0.7554413489601204, + 0.7666209870562611 + ], + [ + 0.7602412409243213, + 0.7753034287397631, + 0.7737938468282183, + 0.7734919654112046, + 0.7612774412993935, + 0.7429701373742656, + 0.7417031380928163, + 0.7556823071775794 + ], + [ + 0.7356422077535445, + 0.7754093812203173, + 0.7588016178670219, + 0.743173810418089, + 0.782577120141176, + 0.7526024722167826, + 0.7382982820295757, + 0.7490228443531554 + ], + [ + 0.7656931211140169, + 0.7582308729639864, + 0.7474664132424788, + 0.7841272496310758, + 0.7606245349396313, + 0.7496140150944477, + 0.7812313278646715, + 0.7574592832757164 + ], + [ + 0.7633718150273545, + 0.7451985213665698, + 0.7596222808702942, + 0.7477744448315442, + 0.7787476971734225, + 0.7527874908761494, + 0.7358337795033768, + 0.7426843579528479 + ], + [ + 0.7621370637454457, + 0.7879020386151, + 0.7964922219891887, + 0.7608066982576122, + 0.7512077038765774, + 0.7592748292272997, + 0.7397459174923856, + 0.7785635067634478 + ], + [ + 0.7448580879391891, + 0.7517478121017902, + 0.7274061141304174, + 0.7830663652004094, + 0.7759444263899737, + 0.7843943922771247, + 0.7558726814584779, + 0.7495719590339897 + ], + [ + 0.7355262167222251, + 0.750231235363392, + 0.7344409576601316, + 0.8076945952804776, + 0.7298323530851526, + 0.7593383039435563, + 0.8103146128842194, + 0.7413597819287329 + ], + [ + 0.782497242057358, + 0.7872971690634394, + 0.7800243578463563, + 0.7667448225965292, + 0.7492372346674934, + 0.7506717278379292, + 0.7270964564290339, + 0.726937806688616 + ], + [ + 0.7386200671878063, + 0.757791520217995, + 0.7797044185606394, + 0.7237302122794526, + 0.742053112419761, + 0.7538415671281887, + 0.7807443876648016, + 0.7616793483680793 + ], + [ + 0.7449066792242496, + 0.7442804389479405, + 0.7398540321583187, + 0.7188376535815366, + 0.7392573254796522, + 0.7557711462778663, + 0.7647365643084186, + 0.7305193353444258 + ], + [ + 0.7562871807430113, + 0.7430602855075573, + 0.7694125446132599, + 0.7724540025337225, + 0.7778695532565784, + 0.7808241194725835, + 0.7295191850111234, + 0.7524891489916873 + ], + [ + 0.7334755248968812, + 0.7676407760144679, + 0.7741889862190746, + 0.7718816662493299, + 0.7715448209719052, + 0.7687154844450238, + 0.7338581636947414, + 0.7499752679243924 + ] + ], + "r2_matrix": [ + [ + 0.9833818826315904, + 0.9873722863521314, + 0.9833252360044054, + 0.9857241844648515, + 0.985407643511392, + 0.9825284725396646, + 0.9845730719734876, + 0.9824954018199186 + ], + [ + 0.9871044674624878, + 0.9806293348082261, + 0.9829065183490789, + 0.9877554078947384, + 0.9880625481807306, + 0.9875745220287557, + 0.988691338478894, + 0.9852381871862419 + ], + [ + 0.9814162206668591, + 0.9852885238380223, + 0.9815076156374826, + 0.9870864697708666, + 0.983222840956547, + 0.9891318555942799, + 0.985260291400293, + 0.9845983969270333 + ], + [ + 0.9888329678426818, + 0.9813949800298651, + 0.9892258932281337, + 0.9814953368012, + 0.9910434087138253, + 0.9814922899818376, + 0.9829391925087182, + 0.9840371849791438 + ], + [ + 0.9814743091424304, + 0.9851891812967897, + 0.9814766501692515, + 0.9894239087195513, + 0.9825644546145994, + 0.9854632524370744, + 0.9822845339269598, + 0.982987707133729 + ], + [ + 0.9865376885331709, + 0.9867309412671889, + 0.9824319345760519, + 0.9869149314035988, + 0.9795831884120356, + 0.9907333811284594, + 0.9738546899783261, + 0.9783739578435492 + ], + [ + 0.9854854082470005, + 0.9887967607054302, + 0.9913357062586896, + 0.9847680271212452, + 0.9933390851869092, + 0.9870032783749626, + 0.98374534107329, + 0.9759247738725854 + ], + [ + 0.983107301660451, + 0.9843058593348069, + 0.9903896356453183, + 0.9827027160152279, + 0.9790395006480171, + 0.9865079282866984, + 0.9825510609730284, + 0.9807662383939311 + ], + [ + 0.9783267270383229, + 0.9828936978527412, + 0.9886430141314158, + 0.9891046024303363, + 0.9771759908970473, + 0.9904708701900017, + 0.9835943852934592, + 0.9858449595085471 + ], + [ + 0.9846949063202687, + 0.9914908282368292, + 0.9812196028421735, + 0.9792161289421317, + 0.989246262693765, + 0.9786592863518, + 0.960811316844447, + 0.9917849465300692 + ], + [ + 0.9836495921530137, + 0.9882008044955353, + 0.9839870880548162, + 0.9812037004425882, + 0.9885459296492869, + 0.9697510797250257, + 0.9754978426714968, + 0.9859181503331989 + ], + [ + 0.9747456621652499, + 0.987820066123344, + 0.9891443743763394, + 0.9677564158180697, + 0.9823258489511612, + 0.9753133628074764, + 0.983170131443609, + 0.9876038569339498 + ], + [ + 0.9773235929613506, + 0.9658433572143877, + 0.9926659512947058, + 0.9790962717090266, + 0.9775019368395595, + 0.9872230128463472, + 0.9912796822859836, + 0.9849898065225227 + ], + [ + 0.9846758517531594, + 0.9866956300590964, + 0.9866262831531937, + 0.9821386537429958, + 0.9722813362777365, + 0.98441619727783, + 0.9917183499494335, + 0.9888404085926427 + ], + [ + 0.9846927557527363, + 0.9841054604577277, + 0.9824974043318593, + 0.9817447803138643, + 0.9754213132929171, + 0.9880974196809147, + 0.9812339535869733, + 0.9889933877543383 + ], + [ + 0.9768602510928369, + 0.9813833126011607, + 0.9882973919216269, + 0.9782631451479746, + 0.9676742814661585, + 0.9901846444173862, + 0.977683098902505, + 0.9916963525756928 + ] + ], + "summary": { + "gamma_mean": 0.7575224192164092, + "gamma_std": 0.01695771074339707, + "gamma_min": 0.7188376535815366, + "gamma_max": 0.8103146128842194, + "gamma_range": 0.09147695930268274, + "gamma_global_fit": 0.7572335446562629, + "r2_global": 0.9854988884554713, + "gamma_mixture_pred": 0.7575224192164092 + }, + "outlier_heads_high": [ + [ + 8, + 2, + 0.7964922219891887 + ], + [ + 10, + 3, + 0.8076945952804776 + ], + [ + 10, + 6, + 0.8103146128842194 + ] + ], + "outlier_heads_low": [ + [ + 13, + 3, + 0.7188376535815366 + ] + ] +} \ No newline at end of file diff --git a/data/exp_gamma_field/EleutherAI--pythia-2.8b_gamma_field.json b/data/exp_gamma_field/EleutherAI--pythia-2.8b_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..9b6cf4fbcebd2a25d6dcdc630dfb7131345e80eb --- /dev/null +++ b/data/exp_gamma_field/EleutherAI--pythia-2.8b_gamma_field.json @@ -0,0 +1,2465 @@ +{ + "model": "EleutherAI/pythia-2.8b", + "theta": 10000, + "T_train": 2048, + "n_layers": 32, + "n_heads": 32, + "d_head": 80, + "n_prompts": 50, + "seeds": [ + 42, + 123 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "gamma_matrix": [ + [ + 0.9684612838575497, + 1.0339178902277308, + 1.0304669195884277, + 0.9025630399608051, + 1.3687096037874804, + 1.24396259449359, + 2.313285377479016, + 1.000992031898423, + 0.984695370822692, + 1.4246425366318887, + 1.1766856563171517, + 1.2538007138202154, + 1.2015797490310784, + 0.9567635878868135, + 1.2068218287928907, + 1.0897116574646446, + 1.4745616076075516, + 1.1813526245611983, + 1.5333066725624642, + 1.227825058908038, + 1.148647376641047, + 2.101466976510533, + 1.2737217100929432, + 1.1996890901330446, + 1.1887376182842218, + 1.3459431196041542, + 1.628381741781929, + 1.0348118850721855, + 1.0523382135933212, + 1.099467632374648, + 2.182775241767506, + 1.0788331294215237 + ], + [ + 0.7985944310824313, + 0.9392909703265551, + 1.0216527802427533, + 0.9536520785956899, + 1.3090611832994878, + 1.3818171817994473, + 1.2491714740920903, + 1.465716800162722, + 1.6372216289452715, + 1.2166707765073743, + 1.5211744523770023, + NaN, + -1.642828489173435, + 0.9803977685259938, + 0.8560676138236117, + 1.3069941116601673, + 1.2569509164365142, + 2.152871047311676, + 1.1155228072777301, + 0.8481036405347139, + 1.438735409383134, + 1.0619459126509965, + 1.116552700187067, + 1.4060726960139185, + 1.0216301689456735, + 1.1209644979834938, + 1.8035068822466684, + 1.0038463942228777, + 0.9976114648152813, + 1.2282897288756287, + 2.1568005377322224, + 1.1224702469811387 + ], + [ + 0.903452493690201, + 0.9009402593537634, + 0.5431785733520859, + 0.49029144334022007, + 0.8313418087363575, + 1.0014133684526036, + 1.0319279692201, + 0.8343524624349722, + 0.8567305988715151, + 0.964145755401422, + 1.0331307103980765, + 1.0668254809123974, + 0.8306034388690494, + 0.8561944805215189, + 0.8336196447241753, + 0.7240972073415548, + 0.8046714575774218, + 0.5821061630042677, + 0.9578572568273022, + 0.9139511406414702, + 0.8689387926541171, + 0.9573302495564368, + 0.9877041859499838, + 0.9942352540857258, + 1.0111374106266349, + 1.25098146879672, + 0.8826174104343514, + 0.8366860868206059, + 0.5450919032371655, + 0.8867623005541511, + 1.0573679647862582, + 1.0260120606015042 + ], + [ + 0.9484206142340419, + 1.1822510437814386, + 0.927859268993627, + 1.0579551939187453, + 1.1842581887381927, + 0.9883512309953391, + 0.9553303290488608, + 1.422149862871486, + 1.78470499032957, + 0.9291185903072884, + 1.1468853431221855, + 0.7556364547656254, + 1.0365531660936596, + 0.7578026830412479, + 1.2783303709954903, + 0.9077867379314561, + 2.0402515532269994, + 1.0417776434509947, + -0.32845012271190555, + 0.8848439931742829, + 1.2460606622094332, + 1.3983080007566875, + 1.4691982765371512, + 1.6326230611416321, + 1.1228529479255425, + 1.0986805149941716, + 1.3108205420419465, + 1.0361630554305181, + 0.9601612305873374, + 1.438911505317102, + 1.014170327116278, + 0.9354923165082059 + ], + [ + 1.3858256068555401, + 0.9333738612675213, + 1.0908752931960661, + 1.0620177410865626, + 0.8766258830127779, + 1.0056490892707932, + 0.9214647535895786, + 0.9444753537279703, + 1.0111424874876984, + 0.9119152816696995, + 0.9176282816833947, + 1.3931096888547738, + 0.9199067328643075, + 1.3063600819590833, + 0.7209889613169231, + 1.2655498875990656, + -0.741772843418082, + 0.6647160000181657, + 1.471595921079923, + 0.8418055610778937, + 1.0729092016202733, + 1.152504054795757, + 1.3967717458804911, + 0.8364918534070889, + 0.6502936027680949, + 0.48023394199631647, + 0.6674567883815532, + 1.479565428046811, + 0.7071720742226194, + 0.9441268476180532, + 1.417708840438199, + 1.0004666764613754 + ], + [ + 0.8990317835469334, + 0.5619866904195896, + 1.1409683167134193, + 1.0381036594346629, + 0.8979590360129178, + 0.4791237785104768, + 1.1935148972707332, + 1.516806543536965, + 0.7696168176894823, + 2.520993423699543, + 0.9528354981847019, + 1.8646620610827287, + 1.3728539714434496, + 1.1430208680770362, + 0.8037023879552962, + 0.9190282256860358, + 1.1076938028415537, + 1.6409214804362737, + 1.3442922917580018, + 1.5506617098066449, + 1.0211158059002001, + 0.9637235910209383, + 2.1474805686546357, + 0.7831243123017609, + 0.7522440917802077, + 0.9332822227959293, + 2.3441380874995583, + 1.249228119469476, + 0.9249963373976251, + 0.7707442241962447, + 0.82294698799603, + 1.6584686837073037 + ], + [ + 1.0375368895208272, + 0.8777157986734184, + 0.8007376134926597, + 0.9603820764611269, + 0.8486546620289035, + 1.1590652951129814, + 1.4086822312625222, + 1.267418663671938, + 0.8584784663609436, + 0.989403799894649, + 1.1135239669269736, + 0.7153241979927288, + 1.4774550993393276, + 1.9248521499697515, + 1.3458240428291022, + 1.23241762764188, + 0.9536617637817119, + 1.074959439205153, + 0.7880173897036037, + 1.2727051368015978, + 1.373924192412394, + 0.6594364866128042, + 0.8706865788901688, + 2.1282756266951934, + 1.0313018996802383, + 0.9022067581152148, + 0.8922876330238798, + 0.6760975367073883, + 0.8702973142494849, + 1.0264081944117605, + 0.8122151654273535, + 0.8984103293300414 + ], + [ + 2.0624160065388915, + 2.0410950859112744, + 1.5393738642832784, + 1.1045929277996083, + 1.331696288283133, + 0.800651323586866, + 1.0335724762841998, + 1.5506385678676853, + 1.1618169157911025, + 0.8697579437994651, + 1.3591047408512567, + 1.1547923398857454, + 0.8194797872523095, + 1.293014433169825, + 1.2755560470091976, + 2.197394798587054, + 1.5206149506863695, + 1.2648717188868794, + 1.2033143129821775, + 1.087096123622408, + 1.7244857291610083, + 1.4926546825545972, + 1.145544999446442, + 0.8967200514273965, + 1.9269509780233145, + 1.0158049066952293, + 0.9428557222099949, + 1.280834345500368, + 1.3424737663496393, + 1.0407635336377832, + 1.4399936141472585, + 1.0696436696152438 + ], + [ + 1.0389740941963022, + 2.4091814700260805, + 1.9596937581032254, + 2.220781186972234, + 1.647825153694857, + 1.272760896932529, + 1.794821201350996, + 1.36300995021975, + 1.0851163833050574, + 2.4871249616198425, + 1.1830705248834972, + 1.180357454980619, + 0.5876163835450922, + 2.571834719646585, + 2.2177458385307087, + 1.6314965977119606, + 1.568988649463521, + 2.1965923078484524, + 1.4603094906775986, + 0.9389933180413018, + 1.9252472799542948, + 1.0215817988597824, + 2.006301358138481, + 1.381775486167335, + 1.3734206555801307, + 1.4704026247373152, + 1.73410910760629, + 1.6806732396240702, + 1.3227870311578986, + 1.1022914343301202, + 1.207008696936931, + 1.211550121633561 + ], + [ + 1.1520689661380643, + 0.9372791380229701, + 1.966621930585006, + 1.6041265202124382, + 1.009422495306904, + 1.2566044580550781, + 1.6298128686955748, + 1.1560110666274988, + 1.090536683643806, + 1.47497761063527, + 1.887396343029293, + 0.9718454460732758, + 1.0472638529602099, + 2.052810777700787, + 1.2320403677445673, + 0.9534017367888746, + 2.1016834000038425, + 1.1122777538196962, + 2.198118573396416, + 1.5861402356020964, + 2.8983114576131994, + 1.3380106945998405, + 1.540339813609844, + 1.9523125953117653, + 1.6864609940737099, + 0.9170837096329637, + 0.7701028411157761, + 1.8210085236234268, + 1.130947789711869, + 1.9157987902626135, + 2.0139575610178237, + 2.16797647923725 + ], + [ + 1.9552600307366517, + 2.0145029718137346, + 1.4354524241255686, + 0.897908405838112, + 1.4914612878842877, + 1.089912844309734, + 2.5655492087542195, + 1.2646152529945134, + 1.1084294814240905, + 1.0025412385716455, + 0.8343592186625433, + 1.7410257195859187, + 1.2022440995120163, + 2.206494641909432, + 1.9028974717601006, + 1.1099573272054013, + 1.56466742171271, + 1.331933611769828, + 1.2020002471323588, + 3.168775396144708, + 2.273749690282262, + 1.1041326849306436, + 1.8012422905017003, + 1.67173767243631, + 1.6998575174988555, + 1.04169236002351, + 2.3213829695983312, + 1.43549146591022, + 0.9465457588643755, + 1.0399738344927059, + 1.2868058165102996, + 1.5786080492301786 + ], + [ + 0.7752691072181269, + 1.1223843038954022, + 1.115949571386597, + 2.279615405584609, + 0.9112780467026117, + 1.953855794076866, + 0.9790139858596096, + 1.9499143325116792, + 1.0357654345712917, + 1.3454498960416623, + 1.2881729282531076, + 1.3567582433752754, + 1.021676754541088, + 0.9234750419205215, + 1.0863923637857564, + 1.2354267510913737, + 2.176666179985887, + 1.5540655899906726, + 0.8522823757260755, + 0.7409525888621101, + 1.4561089076607476, + 1.2707888237147393, + 1.0217641508138868, + 1.1079415282844765, + 1.2116319852225663, + 1.020076212247605, + 0.7693457847233882, + 1.035593733992739, + 0.767155161593545, + 2.092877964779922, + 0.7057871854086843, + 1.3269143725484247 + ], + [ + 0.9526929696823531, + 2.345948769845182, + 1.131559669734078, + 1.4106080475420664, + 1.362520675181652, + 1.5382487624803851, + 1.249913618025038, + 0.5804984719432785, + 1.7441780152930597, + 1.1614978715425346, + 1.276814182110607, + 1.157442242133951, + 1.1461564660846606, + 1.1821883926623504, + 1.0989295699007462, + 1.851206049274701, + 1.0922588381223695, + 1.0424592922762639, + 1.0596117309730975, + 1.4517116670109744, + 1.1666452083939725, + 1.200688900256354, + 0.8869200747220662, + 1.0787403606509904, + 0.723026345660838, + 1.155982168477616, + 1.0969547107648334, + 1.0201659795874114, + 1.0421713214236006, + 0.9974319485403947, + 1.1894268395326295, + 2.360562009798813 + ], + [ + 0.9955210053014873, + 1.1202047852499275, + 1.050978807196498, + 1.228603566107877, + 1.3972635959787538, + 0.9961327659123075, + 1.9398511142955812, + 1.2755351356621543, + 2.6922825436601077, + 1.289246340582551, + 2.019228410107825, + 0.918668414748377, + 0.7275682264739378, + 1.6757887494525725, + 0.9670355430249267, + 1.6927634314126174, + 1.04551281485972, + 1.9687884663257775, + 0.844895998400597, + 1.5634796136178952, + 0.8629936851079221, + 2.009503300073331, + 0.9128951981908019, + 0.8994212899446594, + 0.6740877777079632, + 1.8205511045601699, + 0.8905393573104393, + 2.6157569651008603, + 0.803369473324547, + 1.1561928820865046, + 1.0847143314376244, + 0.9791470251953798 + ], + [ + 0.9181011246634323, + 1.3513890430880808, + 1.1897881795739964, + 2.691993550879899, + 1.167191641449099, + 2.0837650771786915, + 0.8960049644579822, + 2.6557039084908234, + 1.434364079266795, + 1.1720678492480352, + 0.784697519667266, + 0.8912528322142426, + 2.52579198711268, + 1.9288401814249128, + 1.1751628717676534, + 1.8014131087850442, + 1.519917679234904, + 1.5389037318912822, + 2.5363557882221075, + 1.3782791217733334, + 1.0441018267497664, + 1.6354457755546659, + 1.4595280487265538, + 0.8501757307685331, + 1.700761851339979, + 0.8521681729243287, + 1.0285151954140785, + 2.1676471421219943, + 1.209928405695247, + 1.2387848050370753, + 1.6968777769601853, + 1.4366743796486667 + ], + [ + 0.8093078239613855, + 0.6635479154230151, + 0.8094081966950114, + 1.456005477383608, + 1.433076975387793, + 1.913813877719152, + 1.1881692801178183, + 1.9628545051847142, + 1.5002906510585825, + 0.8030734529997532, + 1.3072806633793856, + 1.3474355371981004, + 0.8373284008224956, + 0.7312090656476656, + 0.546280492532167, + 2.5504963013947752, + 1.0444117784719655, + 0.9461803326397081, + 0.9051450195009638, + 0.7809362131207045, + 1.1219837825808001, + 0.8121627480243401, + 0.925862201761323, + 1.098448323079878, + 0.9271688202178193, + 1.2529864236904704, + 1.1317722694688395, + 1.0149133487756097, + 1.3086442969232432, + 1.9514298545425166, + 0.7771624854580714, + 0.8263946234436831 + ], + [ + 0.7900364959987456, + 1.4505060412032922, + 1.0105054524106678, + 1.462721601586599, + 1.1570973970174674, + 1.498265483752507, + 0.8489104716783537, + 1.2350806696019747, + 1.4568092473374528, + 1.8566674051650727, + 1.5683439163536337, + 0.828054695290214, + 0.9258028167359984, + 0.8564198764945683, + 1.0227393364533623, + 1.617486184651825, + 1.229043456302227, + 1.9001447597815941, + 1.6317899116407224, + 0.7836371499257531, + 1.3648516966344884, + 1.1646688915596572, + 0.6623474134009603, + 1.9848628569337254, + 1.167360703426291, + 0.8618957960169176, + 1.4860800595512043, + 1.640679676637105, + 1.4916619288216661, + 1.4798963756691599, + 0.5920790172918108, + 0.7613057687439019 + ], + [ + 0.8721960111387682, + 2.2116711070393666, + 1.08790799784248, + 1.0264376687719252, + 0.9137305441121171, + 1.7031696654901165, + 1.626306284821764, + 1.0895396907801518, + 1.2017395151826356, + 1.0910225702909992, + 0.9016623347289288, + 1.1242594861257873, + 1.0435962319370045, + 1.0339217658785786, + 0.8409059510709588, + 0.6455633948937591, + 0.8950599056562025, + 1.1436110935423442, + 0.9747583489568208, + 1.7434440980987558, + 1.0131336162674116, + 1.219713011444448, + 0.7786713271539245, + 0.9064479972039423, + 0.8909390415241036, + 1.314630978359939, + 1.043942831435297, + 2.557353573544899, + 1.0061022863823217, + 0.7452488208448064, + 1.006089800108538, + 1.3155511618431963 + ], + [ + 1.4571144767506097, + 2.4734103118766484, + 0.691105729676047, + 0.8270623216455715, + 0.6801053318248549, + 2.5028600353070365, + 2.4271383494551726, + 0.7597111880601387, + 0.8404898939697151, + 0.8446472486057148, + 1.1383377712581695, + 1.2300750709053618, + 1.0383375320111712, + 0.8425458919364528, + 1.0125117175628904, + 1.9852463183525966, + 1.2543117531203614, + 0.6787580405559449, + 0.8042823463650224, + 1.4254900173907414, + 1.167930705141909, + 2.1076966749220554, + 0.7549077523922617, + 0.9164324861176777, + 0.8927247252172542, + 1.0458346931234175, + 1.5647621838494359, + 0.9633693030637691, + 1.4103645393688562, + 1.9137475505771173, + 1.399115173603682, + 1.3884283995918147 + ], + [ + 0.8519204314355131, + 0.8792232541372139, + 0.7257172807315508, + 0.7367927185169216, + 1.3431137768344932, + 1.8635702416888629, + 1.6742284871030253, + 0.7978996947615393, + 0.8089128351496203, + 0.5418137778705474, + 1.6910839872151422, + 1.1984288499921534, + 1.100914994689358, + 0.9990046700466023, + 1.7783694306775684, + 1.9432442675940187, + 1.0297500576809053, + 1.0601151931603838, + 0.881434995232208, + 1.1938937833581091, + 1.8875654466322247, + 1.2045573770763134, + 0.8613811710443672, + 1.7182786846510743, + 1.3101704618580998, + 1.5460170276311787, + 1.8708165774590721, + 1.043849344252672, + 0.9981977971015378, + 1.0618737086902006, + 3.0858230304534997, + 0.8640387478512916 + ], + [ + 1.315249751048123, + 1.1234638542769835, + 1.356246807850078, + 0.9882417485184152, + 0.9466550335080721, + 2.046311961997012, + 1.0153487393540146, + 1.203092770884635, + 0.9555640680024637, + 2.1164724639881336, + 0.8021300550636088, + 0.7857496204730855, + 1.0626826235537883, + 1.409375671922079, + 1.5780189556630166, + 2.6401636343884687, + 1.943351114220358, + 1.4095091937680984, + 1.0770270635251256, + 0.8744937238880999, + 2.0010206318106456, + 0.8445424997474186, + 2.114525612261855, + 1.6339231074749754, + 1.3971762724048002, + 0.9765566873205073, + 1.3419040317556867, + 1.4764996148581695, + 1.4967119547886774, + 0.7546493289383162, + 1.140706217458957, + 1.4013175929778952 + ], + [ + 1.8612867071709331, + 1.418536094242061, + 1.5349062399710838, + 1.1155515830155718, + 1.4835127721463903, + 2.1064657889285394, + 0.8791298642134139, + 1.2988161961742388, + 0.8252614328628, + 0.9132471566223952, + 0.9184387595148117, + 2.0532301811297957, + 0.8966581270294203, + 0.8859590643652705, + 0.9421150968697087, + 1.8499334756071264, + 0.9248861640176085, + 1.9811890963039331, + 0.8024962717306814, + 1.1279311894989352, + 1.3357299620713572, + 1.2227709458070548, + 0.22322898914813488, + 0.8331939964663975, + 1.215915775191905, + 0.8713896815497177, + 0.8812069434993025, + 1.7837625096056753, + 0.787909968761836, + 0.9295124589540289, + 0.9729434407985029, + 2.8265982563530914 + ], + [ + 1.7840396492030992, + 2.6529962430735226, + 2.0865347031077297, + 1.071182656077906, + 1.5771998029527283, + 0.825692460910393, + 0.9671949624264369, + 1.0372151160828025, + 1.781632768401282, + 2.5184040525140157, + 0.9971315897864163, + 1.0419002453321382, + 0.8880446584258518, + 0.8104374226321931, + 0.8573224165701709, + 1.1560947573160714, + 1.0663172565082537, + 1.0285737492299707, + 2.785477726827719, + 2.11902359660195, + 0.893429395406002, + 0.6860625352128249, + 2.094457554135542, + 1.8009611070613194, + 0.9470654771687912, + 0.9359561369351949, + 1.1293715007847278, + 1.0092567728504387, + 0.5802612884660605, + 1.0834765604189092, + 1.2680302876147693, + 0.7777178084674182 + ], + [ + 1.4779292404268238, + 1.9792477952344776, + 1.0459282406721206, + 0.8458257319009359, + 1.0798338626355877, + 0.9153174115546113, + 1.328096820120912, + 1.4363538150868689, + 1.1132787038344265, + 2.3050767945701036, + 0.7722267217425904, + 1.9083379384622408, + 1.0542817097426793, + 1.159080101361411, + 1.7583043205789797, + 1.3440530777882083, + 0.8943248544688748, + 2.607363655667063, + 0.7537288601870127, + 2.011138038164642, + 1.8830046219125167, + 0.3544784521346708, + 0.8113446220009722, + 2.414248081822973, + 0.8446978540870832, + 1.077545952317728, + 3.30883919376384, + 0.7749205480298084, + 1.669290771890235, + 1.591140625247877, + 1.1641492972534857, + 1.448985316166156 + ], + [ + 1.2707638589263617, + 1.6675992663133947, + 1.744569042743504, + 1.5372454984534214, + 1.709131855418715, + 1.820423667634265, + 1.7989614856385554, + 0.8253972919005429, + 2.3988244801224914, + 1.7939718629347925, + 0.848595520800144, + 1.62800771928065, + 1.378162818873813, + 0.9503762635623023, + 1.9297126039619978, + 1.1002716007801703, + 1.1971368888348126, + 0.8741580736307467, + 1.7231526502331214, + 2.489591518971722, + 1.7537209308562027, + 2.3551185618768153, + 1.714053562921399, + 1.4301945904176863, + 0.9595929138655208, + 1.2179128740053249, + 2.2514998647223425, + 1.3518217024681372, + 0.909186988012802, + 1.6039818900683107, + 1.838538715365436, + 1.51795004847425 + ], + [ + 1.6257893484689068, + 1.6933986265932774, + 2.066113485805941, + 1.3721769180771681, + 0.8012885003002745, + 0.863592960661768, + 1.2175351242861006, + 0.7016778148589055, + 0.7981234998973519, + 2.2099858422957213, + 1.144041744389356, + 1.8215226032538958, + 1.9567516652384178, + 1.4401677645743929, + 1.0375579196033606, + 1.4809218036122636, + 1.0290294220734946, + 2.900071611039438, + 1.3424925505660794, + 2.717588225962268, + 1.0546207033491377, + 0.7548005791535362, + 1.3471936292424114, + 1.3992992630635215, + 1.8314489644712566, + 1.237248447183687, + 1.243516265162312, + 2.06120370294367, + 0.903157761018015, + 1.5400910558247791, + 1.1415375736820235, + 1.2338333330413933 + ], + [ + 1.488549187695932, + 1.9514100410099608, + 1.0421309740257152, + 2.180891953993847, + 1.7123253181928955, + 1.4725597853909371, + 1.4980422898692878, + 0.37831125271103017, + 1.0118467580824455, + 2.1783717069350352, + 1.4652139511379774, + -0.07795209745529916, + 1.578379223065444, + 0.481612216836707, + 1.6426653622274952, + 2.424052487727612, + 2.1667822364603073, + 2.5945339811933006, + 1.714590080235969, + 0.10970607521004332, + 0.9254055202887493, + 0.7895776711063258, + 1.1673932007406331, + 1.2100864927292139, + 0.9972087404307668, + 0.9661291564189449, + 1.2631238312977342, + 0.6720058374565785, + 1.80556400147551, + 1.3608593110436793, + 0.7183022805824527, + 0.959840693298269 + ], + [ + 1.4324207180623902, + 1.8437278613163486, + 1.390101056288038, + -0.014933489348002694, + 2.0726517969172407, + 0.5347427979719169, + 0.9444340368140993, + 1.0597831456655553, + 0.49754653645181357, + 0.8643979608232668, + 1.1337115003468456, + 0.8857290032156354, + 0.7296730494696303, + 1.3827601361054653, + 1.5121366641573084, + 0.7909196425744597, + 1.3916987594287284, + 1.5216900786066074, + 1.3312542684671917, + 0.6347531303820436, + 1.706402937185419, + 3.004785169405879, + 1.665181722037025, + 0.5385725030184797, + 0.9559615368781986, + 1.4133958127256667, + 1.1652689824301596, + 0.7668144675529833, + 1.5513174486133312, + 0.2548671408240638, + 0.816130951445994, + 2.7066743734410044 + ], + [ + 1.5281533522166733, + 1.0709412408210996, + 1.0785343700535006, + 1.6386124765729968, + 0.9662506762220902, + 1.5201479208565336, + 1.2743539829626234, + 1.5007367992853884, + 1.4226831723800615, + 0.5594804774737218, + 1.9179894226385013, + 1.2707294956953896, + 1.07196304064161, + 1.5790771572623419, + 1.686829008622297, + 1.737750617537793, + 1.5273154779882743, + 1.2402707581479187, + 0.5868829273176749, + 1.7557256145705837, + 1.375529630618217, + 1.4946674832154871, + 1.9293373028648209, + 2.1995234582204093, + 1.662342347408856, + 0.9204548660215242, + 1.0585413546604452, + 0.503797432552394, + 1.4409120143185667, + 1.3632107177429242, + 0.776048847717784, + 1.9535452091012486 + ], + [ + 0.8078712168294141, + 2.1490452101883846, + 1.7341662706592733, + 0.6313315472862863, + 1.2810120552132345, + 2.373884356802185, + 1.9580719528363992, + 1.9984786386147346, + 0.7471300543609669, + 1.7500537685681672, + 0.5410774290425306, + 0.8417635319589551, + 1.251026945050826, + 0.5238199064207247, + 3.2042816117163175, + 0.5982843699336738, + 0.5727539353914193, + 2.279888043651013, + 2.1314512060609077, + 2.328746682779298, + 1.2460579364547824, + 0.41573513949824237, + 2.6109390972501783, + 2.9724816136358934, + 2.051105719934839, + -0.1843210132287975, + 2.675290041796798, + 1.049089999486508, + 0.9006625540901928, + 2.246649992600466, + 1.264679731471379, + 3.640838262903864 + ], + [ + 7.389473235765624, + 2.0812290060497385, + 2.783810418683214, + 1.1748748704476422, + 1.5408089011761694, + 0.5692491797746277, + 2.0769382201296738, + 0.8620199869532238, + 2.049703429959011, + 1.4083317561135253, + 1.8444329540216557, + 2.0310573399178313, + 1.9585962185458747, + 1.7532801650104244, + 3.686142246547189, + 0.15818208267566625, + 1.8250834099696716, + 2.844147171060353, + 1.5201744279676452, + 1.5919404590448074, + 2.7162351913589973, + 3.0448343562077067, + 2.1293453188159472, + 2.775651741732271, + 2.898319807402979, + 1.980679036394206, + 3.97955197289806, + 0.46947309370667845, + 2.363253187396666, + 1.2289616386142064, + 5.290577310409102, + 5.578895816468481 + ], + [ + 0.6945625123356826, + 1.0568738242924858, + 1.0233777496008103, + 0.05812207130765665, + 1.3038256251317417, + 0.5248238336220878, + 0.7593329320369706, + 0.8801711802479031, + -0.23978330047035096, + 0.4433307586901203, + 1.2597701928064606, + 1.3084999655631109, + 0.14225184853250525, + -0.27314413982530633, + 0.1622150688341626, + 0.7925594529830533, + NaN, + -0.04217498789509575, + 0.6473133473265177, + 0.21160419080407658, + 3.1339664249258727, + 0.23869682357367975, + 3.4567578110530595, + 1.206227247077663, + 0.7287061525788223, + 0.7820133192241943, + 0.5814880312194156, + -0.24031267551163857, + 0.6689742831225569, + 1.5758424269012987, + -0.03958358827463807, + 1.6035566768506262 + ] + ], + "r2_matrix": [ + [ + 0.9827769140206407, + 0.9090729276116064, + 0.9558274252342635, + 0.8482725534930942, + 0.95012364912757, + 0.9809992291809739, + 0.9434455334006997, + 0.9804102031966666, + 0.9870864006948248, + 0.986753675249816, + 0.9710176132937485, + 0.9244014866945255, + 0.9881033281698653, + 0.9730003315183806, + 0.9624328855910039, + 0.9686138806402778, + 0.9104266454663567, + 0.9723413349979916, + 0.9646581477409721, + 0.967592399449356, + 0.9851531234679114, + 0.9097927357177463, + 0.9811178858292189, + 0.9805257046876339, + 0.974977715379133, + 0.9647749687077373, + 0.8437739273691651, + 0.9739919521072915, + 0.9760336040014599, + 0.9945811018271696, + 0.9709686785283139, + 0.9737507640950576 + ], + [ + 0.9137466021325104, + 0.9888976074700644, + 0.9618953689418215, + 0.7121843456724912, + 0.98492167121448, + 0.9621652540394664, + 0.2631958615481661, + 0.8929656443682683, + 0.9401860371233964, + 0.6413911742422109, + 0.7835276604570237, + 0.0, + 0.2312539969386811, + 0.9629026690637618, + 0.8871780777384843, + 0.5256328493470956, + 0.5628432250314124, + 0.8899200515236109, + 0.5045053251272082, + 0.8369221626704845, + 0.9782148839287027, + 0.770889942926482, + 0.943533364311092, + 0.7191419847535719, + 0.2860625583883325, + 0.35845133403529494, + 0.7875666809439614, + 0.8791894846078896, + 0.9378116980298171, + 0.8170160330682182, + 0.7652189799491262, + 0.9446626015959766 + ], + [ + 0.945128214128295, + 0.9682381485365865, + 0.39749770314283706, + 0.31783420213135594, + 0.9736745336342201, + 0.9863904180083026, + 0.9955438982284868, + 0.9349685570242889, + 0.9673971680813619, + 0.9797048063569885, + 0.9162063425553897, + 0.9084696126219275, + 0.9583505678184109, + 0.9168892082990187, + 0.950638323521143, + 0.6450975540838082, + 0.9408224003651171, + 0.49583552614428117, + 0.917883413075036, + 0.9881184012049201, + 0.9729493215601347, + 0.9226220914879907, + 0.9811026711998909, + 0.9816346516344856, + 0.9396651069081674, + 0.9759696632844548, + 0.9819229620669427, + 0.8997306312281984, + 0.9142214500067531, + 0.8958848891754864, + 0.9850032474064587, + 0.7833904353837545 + ], + [ + 0.9632499063337479, + 0.4578674381136404, + 0.8840974740297127, + 0.9916416989164799, + 0.6405830439291635, + 0.9775178332490434, + 0.420283517192631, + 0.8425906155965236, + 0.6441442499594743, + 0.7967975795676718, + 0.8561349446968732, + 0.5081954794307451, + 0.9659617119435212, + 0.8929907014570992, + 0.7658240651505278, + 0.8823440468368049, + 0.6670069746235734, + 0.7947204196670583, + 0.1230965854762105, + 0.6080676113828715, + 0.9110595714877503, + 0.6325705930877121, + 0.5997297768296358, + 0.8390701252081756, + 0.500010319522607, + 0.899495734069753, + 0.6427305639085203, + 0.9703895919305422, + 0.982432875977882, + 0.9630665783606418, + 0.9847435107297541, + 0.9090287063110517 + ], + [ + 0.8973290179852769, + 0.8587248176916541, + 0.8623620481504801, + 0.9592486748112932, + 0.9041665300364355, + 0.8814714573269196, + 0.7690385431877853, + 0.8797909171585289, + 0.6953543181314101, + 0.9730815572142246, + 0.4849332840905767, + 0.9061938191216007, + 0.8417278347182077, + 0.9139933970637395, + 0.3261284955927457, + 0.7677407579318155, + 0.11959074304269601, + 0.31480968088653694, + 0.7030935359896555, + 0.9852163580938148, + 0.65901695820251, + 0.9308639053204995, + 0.4519934889966356, + 0.8270078162008787, + 0.14677788577394546, + 0.17840014655773295, + 0.7878498936306082, + 0.8556307939423986, + 0.9715085554391513, + 0.9785614990995265, + 0.9867523996902436, + 0.832553898074859 + ], + [ + 0.9542376753777609, + 0.8614354435119168, + 0.5855639187019444, + 0.943981255680555, + 0.6121241000129681, + 0.27891747793376, + 0.5821811846231628, + 0.6662782477944493, + 0.9539241267741341, + 0.8383240776329698, + 0.93034598217455, + 0.3555546920682333, + 0.7275097839415439, + 0.38666391969141356, + 0.548499117829347, + 0.9742989799045869, + 0.472230153436516, + 0.9179737466304296, + 0.756572979598881, + 0.5921117275051988, + 0.9714730078915372, + 0.9347925300241857, + 0.960997380565745, + 0.9451597188954501, + 0.28983093114812575, + 0.822780923802594, + 0.8258919770189962, + 0.912164864997058, + 0.9349577323749103, + 0.8809158276329677, + 0.772068048605643, + 0.9671079441087135 + ], + [ + 0.8831257357114104, + 0.9208239982845529, + 0.7676674089723734, + 0.9911152120358278, + 0.918650032465829, + 0.9320286196695005, + 0.23922101991628364, + 0.9379298821138188, + 0.9877129188469922, + 0.9473556740838613, + 0.7664996243793789, + 0.8726135292144761, + 0.6956518012442781, + 0.7878370768037157, + 0.7435493246335683, + 0.8576803649799314, + 0.9543701365274669, + 0.5159913242146403, + 0.9811195679338449, + 0.9142920963357147, + 0.8404907022481038, + 0.8483360825394582, + 0.9392074557956117, + 0.8130791412265463, + 0.9793884387353755, + 0.906684058903271, + 0.7848569007276733, + 0.784268814167066, + 0.9081867072707547, + 0.9396065403831108, + 0.9033158492967821, + 0.9700562538049434 + ], + [ + 0.6254747307047719, + 0.8928944780221356, + 0.8826304179655122, + 0.971277931991892, + 0.8993705771785381, + 0.9536725034020304, + 0.9458975125537009, + 0.883841567497615, + 0.9464972629459598, + 0.9729214804391619, + 0.9706690724470385, + 0.9420711906617675, + 0.8326517324976176, + 0.8452005087722948, + 0.9330702035402173, + 0.7490585221401054, + 0.792309434603023, + 0.9541946435182493, + 0.9302619294289767, + 0.7998214770629686, + 0.8392516785852195, + 0.8520479177483081, + 0.9709814729902576, + 0.8639799895701046, + 0.8667542742333503, + 0.4796209416537933, + 0.7494609698088474, + 0.9011814063747099, + 0.9156151335669747, + 0.9343638771730972, + 0.8622286693660827, + 0.8270747231149191 + ], + [ + 0.9197147997875411, + 0.9312198583955379, + 0.727648466160565, + 0.714860331105847, + 0.9303905291008998, + 0.9293840737697174, + 0.9423140977326965, + 0.9438921620043792, + 0.9650188771116881, + 0.8106241756882411, + 0.761765481126718, + 0.9126080740799728, + 0.12033946745701674, + 0.3509820630823204, + 0.7452170674373507, + 0.6577172784404259, + 0.6434028798623006, + 0.7282332776327767, + 0.8779821703833915, + 0.9021055108469388, + 0.5047197748793295, + 0.9687442416836846, + 0.7226457147463479, + 0.9449535550892373, + 0.7637306499027124, + 0.5727693576642114, + 0.8421423036324629, + 0.8131126029710735, + 0.9642632258440719, + 0.9631799824469953, + 0.9657772236569465, + 0.9715713451830249 + ], + [ + 0.9137544407901065, + 0.9192338072433488, + 0.8350905091162888, + 0.9122760940438762, + 0.7918792939045924, + 0.9394482531618422, + 0.978658464517716, + 0.9158730456767068, + 0.9527710556692378, + 0.9511191493343057, + 0.96226639667283, + 0.8472683964802589, + 0.9586134101262733, + 0.9374350407752975, + 0.9662269168474161, + 0.899969210453068, + 0.6393693206280286, + 0.9410237519495797, + 0.7734693088649754, + 0.5469668215328578, + 0.84163722451316, + 0.7808378135248664, + 0.9330966935319656, + 0.9273472204579978, + 0.39546904899862323, + 0.9880523281806293, + 0.4172953277957805, + 0.5282137284373847, + 0.813745037840976, + 0.8604632971157153, + 0.7002184402811813, + 0.8553343352155239 + ], + [ + 0.8882018955467841, + 0.9100265036364257, + 0.8693230566242353, + 0.962842576307158, + 0.9218838589796677, + 0.9502242875941681, + 0.7863858017973806, + 0.9279121966592122, + 0.9604279803192661, + 0.9583268386882658, + 0.48611320544864023, + 0.9613894158162343, + 0.9805660998644425, + 0.8240929114224094, + 0.7864988267349258, + 0.905916648451208, + 0.9380966897629841, + 0.8475225351585456, + 0.8437012369412737, + 0.8621162797388012, + 0.6907547591247747, + 0.45122561698780284, + 0.7365801448459406, + 0.9361589492839688, + 0.8463450352247992, + 0.9523740675236684, + 0.9190134227452438, + 0.4348682709248691, + 0.8976221032611189, + 0.886166291254719, + 0.8534123159482949, + 0.7734412393401087 + ], + [ + 0.6256728679566359, + 0.9133894031037497, + 0.8724040293748815, + 0.754738122190237, + 0.8915783285227102, + 0.8494267720379413, + 0.935122476124106, + 0.8934974705798978, + 0.9448555234194546, + 0.7600215320998488, + 0.9051258198276472, + 0.8689542088576045, + 0.8139745831116506, + 0.9791094396002415, + 0.9083204600670736, + 0.9569549031230559, + 0.9065029996581719, + 0.9102202035070756, + 0.9001736137595735, + 0.8676264350313311, + 0.9384518644233057, + 0.946393499636, + 0.9095335550129733, + 0.9795159403084941, + 0.911472613205628, + 0.9361762250253995, + 0.9124898065784955, + 0.9585737088153474, + 0.6921334590748589, + 0.9061748267691627, + 0.8893409851914421, + 0.9176197913938346 + ], + [ + 0.9416310591831853, + 0.8610274805978081, + 0.8274735768447141, + 0.8335941915726881, + 0.8753605762149084, + 0.8910399182131272, + 0.970000631947812, + 0.8219034577091049, + 0.8956554765217737, + 0.9414785733622283, + 0.9196000430847323, + 0.9736967145872504, + 0.9255984413994558, + 0.9648035977204921, + 0.6458625888952456, + 0.7221565123722438, + 0.9447220384126828, + 0.9118694539939153, + 0.9347787371406336, + 0.8941218352149255, + 0.9934518408249737, + 0.9718179789170872, + 0.9297944359338198, + 0.8987428826979158, + 0.7488181358177559, + 0.9643037669849053, + 0.9232639697382242, + 0.7161015316272148, + 0.9091405407659975, + 0.9034694662914942, + 0.9565923004113609, + 0.7987821816886924 + ], + [ + 0.935444383125328, + 0.9484797664324915, + 0.9127357190142782, + 0.9451803553987301, + 0.9118204691702795, + 0.944855169751169, + 0.9082756202325759, + 0.7719089476948022, + 0.8960763506329724, + 0.966255640533275, + 0.9031875405269872, + 0.9503208189104217, + 0.9312926807363802, + 0.821932436058441, + 0.8599603472982662, + 0.878368398285749, + 0.8725664291736025, + 0.8797009830725883, + 0.9589233128925402, + 0.9689049220973704, + 0.971248542253872, + 0.8693969409399009, + 0.9499673702037585, + 0.9834959527965788, + 0.8831207437065058, + 0.8124619089774281, + 0.7034994301721382, + 0.8866627475187723, + 0.9449525730173699, + 0.9754682265038466, + 0.9338841759226065, + 0.7948557508437549 + ], + [ + 0.7614892950249481, + 0.959350264561395, + 0.882796089493412, + 0.8842446476358095, + 0.9904970948013133, + 0.9371517708292181, + 0.7733129409395, + 0.8842165621150918, + 0.762691074013443, + 0.98653899333751, + 0.6831186733308231, + 0.9689675668915606, + 0.9106150468623467, + 0.8831156941072751, + 0.9579132148033158, + 0.863037420110945, + 0.6456840101522934, + 0.9207733424518034, + 0.8346122324367544, + 0.8574610126052968, + 0.7008858035571779, + 0.7128416980439778, + 0.9365373817931475, + 0.8905425171221943, + 0.7731674894486141, + 0.7916361946071043, + 0.9022713237643524, + 0.8563459866292769, + 0.9377217464923528, + 0.8294964620168378, + 0.8964535333261562, + 0.9472600054236405 + ], + [ + 0.8723211712000021, + 0.6351224591229436, + 0.8963036556231456, + 0.9425882795738477, + 0.9477639390720508, + 0.661619663578042, + 0.8956489523987339, + 0.923992595571537, + 0.8514763817564743, + 0.8851851045962709, + 0.9105714765268722, + 0.8828267847718503, + 0.7667492644588447, + 0.8617793855113526, + 0.7502043973847956, + 0.8634238684186993, + 0.8612547737188289, + 0.7305465188156376, + 0.5294184995801328, + 0.7862158257688512, + 0.849569954114681, + 0.8344130652058281, + 0.9763102357947185, + 0.9132909787545453, + 0.7643935489435729, + 0.9777105834926361, + 0.9009839055733129, + 0.8375297690214677, + 0.8797231100522366, + 0.8670362028313762, + 0.9362944960013196, + 0.8050506105669835 + ], + [ + 0.7831354705518608, + 0.9332981010807071, + 0.9234017691519687, + 0.636417671496684, + 0.960066658958405, + 0.7304085617657692, + 0.8910114335962991, + 0.9443307381079651, + 0.9227004143231994, + 0.7495192143071303, + 0.8369833121229199, + 0.8434178978868779, + 0.9163542141528144, + 0.9661533475453873, + 0.8850963758268717, + 0.9266979112282795, + 0.8345588844195535, + 0.8718240241752054, + 0.7703082834120308, + 0.8504475484691839, + 0.9312075034533152, + 0.8588161724435563, + 0.8745964295513821, + 0.9797460103187392, + 0.8854893361478797, + 0.8211897679857665, + 0.81271006732347, + 0.9058414438295522, + 0.9435221823582863, + 0.8584390162216856, + 0.927835635886813, + 0.7674457451498244 + ], + [ + 0.9437203706754048, + 0.9577260532421444, + 0.9487617072139446, + 0.8636477054247411, + 0.8259980699566213, + 0.9483281069237851, + 0.8265236165077875, + 0.855252067641435, + 0.9251045924330131, + 0.9741741301407056, + 0.9053443202901478, + 0.8212985160430514, + 0.8840149968444049, + 0.8851777678650845, + 0.8875575743514261, + 0.828221321478111, + 0.8891303136753077, + 0.879674417547212, + 0.9684982962327707, + 0.8930376836347967, + 0.820770469225102, + 0.9149929704193702, + 0.868231152938533, + 0.9383987941296531, + 0.8523649190238523, + 0.9157345292036742, + 0.8751941095540083, + 0.8939328306011833, + 0.9008864463925381, + 0.9886251098181589, + 0.9371231065087828, + 0.9268045041845421 + ], + [ + 0.8810488591021763, + 0.7721322917867645, + 0.9153560303116206, + 0.9206441294047749, + 0.8782520908102717, + 0.9436481246089247, + 0.6928986078112154, + 0.8638742542813471, + 0.9251526041373604, + 0.9671278942819225, + 0.9198129793774025, + 0.9188854785379635, + 0.9364754283397518, + 0.9189657493922455, + 0.8893216630297864, + 0.9044968248036698, + 0.7802865412184797, + 0.9226771422898478, + 0.9054590745670315, + 0.6817257566430845, + 0.9247793117508772, + 0.9058315854181217, + 0.7389241678600116, + 0.9795501092516892, + 0.6974723618762652, + 0.8120697757794284, + 0.890683112438558, + 0.9976339320535342, + 0.842222458447625, + 0.9251428288672018, + 0.9034956882132211, + 0.7682178990130495 + ], + [ + 0.8912172701271424, + 0.8124277436605911, + 0.925402974799592, + 0.7696508888689422, + 0.7867640812239146, + 0.775388436333065, + 0.9075213923862118, + 0.8370254636484122, + 0.8069174830379416, + 0.5895879988475536, + 0.900981226171744, + 0.9707264972241177, + 0.9595475650557799, + 0.9618770765064704, + 0.7794358430065018, + 0.8725501543022645, + 0.8777261217627286, + 0.8890755176138084, + 0.9282662667856374, + 0.9044329167190779, + 0.8574151121714655, + 0.7578158587016889, + 0.945973530672579, + 0.8441537043718156, + 0.9269170468326966, + 0.898062239519609, + 0.8896161481402269, + 0.7928572612896254, + 0.8793846873764716, + 0.9414845823325151, + 0.829656455237307, + 0.9429970158876307 + ], + [ + 0.9798893354200272, + 0.9047938827075473, + 0.8424411834624039, + 0.9657082322115935, + 0.9115234623527023, + 0.7262756007476917, + 0.9305070399233052, + 0.9656559425661999, + 0.8557322370400344, + 0.8945343179267851, + 0.8939172697741443, + 0.7947655994115799, + 0.9645809390975444, + 0.8403410814888663, + 0.8678573360727831, + 0.9072885273936021, + 0.8290084421896893, + 0.8843693520559179, + 0.9343259098748533, + 0.9600701382282694, + 0.847322427880705, + 0.8541898467154037, + 0.7885659945936827, + 0.9000752362481127, + 0.9177408709980134, + 0.9001018968028476, + 0.9439916737792821, + 0.8507249581231678, + 0.4485488683314668, + 0.9145270883548168, + 0.8904832266239654, + 0.926189320173752 + ], + [ + 0.8338259629951912, + 0.9305485651043477, + 0.8430065359063259, + 0.90830170905528, + 0.9464977465247765, + 0.8854767984063121, + 0.8798505576785924, + 0.8981981900811433, + 0.9510266254861337, + 0.91662611244921, + 0.8749519829405257, + 0.9371861208320469, + 0.9449865261174162, + 0.9023778060275405, + 0.8035651922910105, + 0.42872254429641876, + 0.9762892673805093, + 0.8996126858566933, + 0.9250646734518644, + 0.6934208409323354, + 0.9274474040402102, + 0.8302300991802225, + 0.07114369277489474, + 0.7962835655039806, + 0.955020771485809, + 0.5707954521734998, + 0.9567017032183999, + 0.827786822438523, + 0.9517759695789627, + 0.8102900394294388, + 0.9136556498632044, + 0.7714949830463635 + ], + [ + 0.7839114892005724, + 0.9338433256660335, + 0.9036198561058133, + 0.9500124968087166, + 0.7826861781905052, + 0.9285148435529621, + 0.9475529149579774, + 0.9243002313719487, + 0.9006106791180996, + 0.6474099651835906, + 0.9190333584172554, + 0.7344111066454249, + 0.782181311437598, + 0.7653802198140417, + 0.9243235018988621, + 0.9566447169769878, + 0.9441437267493308, + 0.923188361120186, + 0.7509657503620386, + 0.7296323449009509, + 0.8851615137809591, + 0.8974113941084612, + 0.8937756395801117, + 0.9151382725038144, + 0.8737027888951396, + 0.9256487905386088, + 0.9526636356003003, + 0.8779781175418453, + 0.5240925730595518, + 0.8794643692306108, + 0.9340151506879012, + 0.7744216781039981 + ], + [ + 0.9092096232506737, + 0.883142052619757, + 0.9789603856199185, + 0.7998688850256028, + 0.8867713955879639, + 0.9979427271567243, + 0.8206717835078777, + 0.9129802725288383, + 0.8083239110277263, + 0.8848654540391793, + 0.3717951928288348, + 0.7630954742324059, + 0.94818954213234, + 0.8260157504739217, + 0.8429536368758046, + 0.8724337306967552, + 0.797928324990184, + 0.8813890440398666, + 0.8489575698172156, + 0.8756222759092954, + 0.5381973740480133, + 0.27358216191524587, + 0.8820264589402904, + 0.7690931171198151, + 0.8480850124739155, + 0.9628221713466633, + 0.676900905029176, + 0.9000880645913383, + 0.8045492988777959, + 0.932023284837094, + 0.834717525620654, + 0.9196304138236919 + ], + [ + 0.866180821924292, + 0.7007741694364868, + 0.6400133010802518, + 0.7319138534314166, + 0.8461722967005201, + 0.6303891662564064, + 0.720394375275011, + 0.8474721594810833, + 0.7144050896868481, + 0.857764675977519, + 0.7241349768040475, + 0.9376649236615203, + 0.9657616643509925, + 0.9478895815193993, + 0.8238553655260558, + 0.8279836576969192, + 0.8127050875636548, + 0.952198392540755, + 0.8847684324207453, + 0.8171626413946382, + 0.8918624418403109, + 0.7309652614757559, + 0.7499349041355692, + 0.952976402542498, + 0.9224416557660348, + 0.886616526696283, + 0.7142299195097208, + 0.8758505876636385, + 0.7010632243771802, + 0.7504881898118781, + 0.7045333402629725, + 0.8885300107869002 + ], + [ + 0.856326322904752, + 0.8611263821012722, + 0.9539045533137273, + 0.8689826255884747, + 0.7748860041244391, + 0.7949162222141822, + 0.9874331362673939, + 0.9814612680983252, + 0.9189103537611355, + 0.7611690273376042, + 0.6893074383936615, + 0.8395670473692916, + 0.8945937663772237, + 0.8658052475206827, + 0.5667714742499206, + 0.9668208364660282, + 0.8749418782388984, + 0.7519561320802365, + 0.8720552385090511, + 0.8434317573932144, + 0.5221067659507492, + 0.47690867161388517, + 0.9152414531313504, + 0.8354502528058209, + 0.8303111242073855, + 0.841751988472463, + 0.5435982457925425, + 0.7949826550290259, + 0.9561503569053071, + 0.8546027323364692, + 0.9566693197054109, + 0.7776343426435809 + ], + [ + 0.7917498411805771, + 0.9368110250805265, + 0.8061120134956421, + 0.7730746429567458, + 0.9019506463678505, + 0.8399036126155712, + 0.7842374694198975, + 0.2581150066146186, + 0.8482748880726059, + 0.9079427959162907, + 0.7095353300782034, + 0.004437445883555546, + 0.7735669013903665, + 0.14997055973305407, + 0.8803093541946779, + 0.8135761550814613, + 0.897493138840609, + 0.8521519509317836, + 0.9402148890259233, + 0.013145329435495512, + 0.5411988020787972, + 0.5892465637234453, + 0.7298814223714857, + 0.8945058700725481, + 0.7968153317599965, + 0.6357208947384154, + 0.5927058905703839, + 0.4433741900930329, + 0.9204603297960375, + 0.34988056691695624, + 0.22084664165898382, + 0.906560454479595 + ], + [ + 0.7839527364049657, + 0.8625024362807249, + 0.6731325959777413, + 0.000280564141519668, + 0.8267521103661649, + 0.5365554985092597, + 0.6876669042040109, + 0.9381160516468854, + 0.4687414596101067, + 0.8583071631307303, + 0.8680352851128299, + 0.9039092475901646, + 0.17246152183858343, + 0.9280668738383531, + 0.8983544094764073, + 0.9619251712768118, + 0.8649581957258335, + 0.9435171015831847, + 0.9594200633809128, + 0.22740009978821696, + 0.9483059005703233, + 0.7080786716808891, + 0.7915838410665275, + 0.2880200051205123, + 0.8078191398329823, + 0.8630491890176937, + 0.8149324601221953, + 0.730748418116537, + 0.7299913237971427, + 0.10830783420806656, + 0.5244490870271503, + 0.8733618409131881 + ], + [ + 0.8775174930486026, + 0.8324149960025309, + 0.9703811648667772, + 0.9437937220268078, + 0.8710781044487031, + 0.7623672457432473, + 0.6364904312998307, + 0.9458652466452169, + 0.7670092064311536, + 0.6775290060965274, + 0.9348508818837399, + 0.8501369831259378, + 0.37354694218514206, + 0.8724172958812777, + 0.94466252814595, + 0.917225173581123, + 0.8724209569665746, + 0.8395870727789874, + 0.6141411114712962, + 0.7423610765754167, + 0.8634397368229073, + 0.8983541637586199, + 0.8697252673424564, + 0.8745694930973534, + 0.9641705895885319, + 0.4930925729080846, + 0.8968749382752766, + 0.14750197800024545, + 0.820550895998629, + 0.8402741512434603, + 0.8692523676395925, + 0.9809048416162174 + ], + [ + 0.6882412362577276, + 0.7556322005974943, + 0.8285584820488681, + 0.32575057506518046, + 0.7557465071886437, + 0.5867817195014502, + 0.867994657317112, + 0.7493460766281519, + 0.456926380107896, + 0.9950963270531498, + 0.3018438654190049, + 0.9801129122498922, + 0.6623640824064694, + 0.45852649936888723, + 0.7862450788489947, + 0.22796149410039523, + 0.6571886951456767, + 0.8533236778984233, + 0.8005720499774214, + 0.39345501339979094, + 0.5122865040247873, + 0.36029997236570066, + 0.5967492604597402, + 0.7759096974195248, + 0.6128843150239822, + 0.010587391635589638, + 0.5762316342020422, + 0.5883680827316484, + 0.757807420517039, + 0.7856326020897475, + 0.17364744536399335, + 0.9207704448178592 + ], + [ + 0.4785664806429404, + 0.3132239293944674, + 0.6051550638116752, + 0.6589812744770651, + 0.8889253906595299, + 0.9500812898539889, + 0.8231244849863986, + 0.9030179725649984, + 0.6230244810571803, + 0.2587907473009077, + 0.4446538522400012, + 0.7706945401843928, + 0.8241279833801795, + 0.6405057717428668, + 0.833475157729816, + 0.006004689253263362, + 0.6565332971312281, + 0.6548422536256622, + 0.9368264595464983, + 0.7836660995856638, + 0.8399154178000463, + 0.6732480511064648, + 0.5343092960404718, + 0.7732501082320589, + 0.5338322492304992, + 0.5818375086140402, + 0.6270368154966658, + 0.953144687413404, + 0.6078666098534292, + 0.6087281004113214, + 0.8181731628642613, + 0.6763452629021989 + ], + [ + 0.8359300532279673, + 0.3168723245600308, + 0.697246275292605, + 0.022335036074681902, + 0.9484005614967804, + 0.351334468983845, + 0.22004085069314683, + 0.3259746417194225, + 0.12325700843627851, + 0.38678409857417717, + 0.39353511113471695, + 0.6922221456982458, + 0.02598085073995915, + 0.014396956448752496, + 0.11121209800488507, + 0.3369775684565821, + 0.0, + 0.002114081203925311, + 0.23632899552559106, + 0.1315817792482573, + 0.8363401410764455, + 0.016918222075056222, + 0.6669166241814707, + 0.37665003222621696, + 0.15585019675365208, + 0.7428476024783951, + 0.19441150008591668, + 0.042710157831615025, + 0.4643094067641609, + 0.5442921503532816, + 0.00035968057548252386, + 0.5771680559884393 + ] + ], + "summary": { + "gamma_mean": 1.3091323429752457, + "gamma_std": 0.6311288030282554, + "gamma_min": -1.642828489173435, + "gamma_max": 7.389473235765624, + "gamma_range": 9.032301724939058, + "gamma_global_fit": 0.9692147759308066, + "r2_global": 0.9722026711765348, + "gamma_mixture_pred": 1.3091323429752457 + }, + "outlier_heads_high": [ + [ + 8, + 13, + 2.571834719646585 + ], + [ + 9, + 20, + 2.8983114576131994 + ], + [ + 10, + 19, + 3.168775396144708 + ], + [ + 13, + 8, + 2.6922825436601077 + ], + [ + 13, + 27, + 2.6157569651008603 + ], + [ + 14, + 3, + 2.691993550879899 + ], + [ + 14, + 7, + 2.6557039084908234 + ], + [ + 19, + 30, + 3.0858230304534997 + ], + [ + 20, + 15, + 2.6401636343884687 + ], + [ + 21, + 31, + 2.8265982563530914 + ], + [ + 22, + 1, + 2.6529962430735226 + ], + [ + 22, + 18, + 2.785477726827719 + ], + [ + 23, + 17, + 2.607363655667063 + ], + [ + 23, + 26, + 3.30883919376384 + ], + [ + 25, + 17, + 2.900071611039438 + ], + [ + 25, + 19, + 2.717588225962268 + ], + [ + 26, + 17, + 2.5945339811933006 + ], + [ + 27, + 21, + 3.004785169405879 + ], + [ + 27, + 31, + 2.7066743734410044 + ], + [ + 29, + 14, + 3.2042816117163175 + ], + [ + 29, + 22, + 2.6109390972501783 + ], + [ + 29, + 23, + 2.9724816136358934 + ], + [ + 29, + 26, + 2.675290041796798 + ], + [ + 29, + 31, + 3.640838262903864 + ], + [ + 30, + 0, + 7.389473235765624 + ], + [ + 30, + 2, + 2.783810418683214 + ], + [ + 30, + 14, + 3.686142246547189 + ], + [ + 30, + 17, + 2.844147171060353 + ], + [ + 30, + 20, + 2.7162351913589973 + ], + [ + 30, + 21, + 3.0448343562077067 + ], + [ + 30, + 23, + 2.775651741732271 + ], + [ + 30, + 24, + 2.898319807402979 + ], + [ + 30, + 26, + 3.97955197289806 + ], + [ + 30, + 30, + 5.290577310409102 + ], + [ + 30, + 31, + 5.578895816468481 + ], + [ + 31, + 20, + 3.1339664249258727 + ], + [ + 31, + 22, + 3.4567578110530595 + ] + ], + "outlier_heads_low": [ + [ + 1, + 12, + -1.642828489173435 + ], + [ + 3, + 18, + -0.32845012271190555 + ], + [ + 4, + 16, + -0.741772843418082 + ], + [ + 26, + 11, + -0.07795209745529916 + ], + [ + 27, + 3, + -0.014933489348002694 + ], + [ + 29, + 25, + -0.1843210132287975 + ], + [ + 31, + 8, + -0.23978330047035096 + ], + [ + 31, + 13, + -0.27314413982530633 + ], + [ + 31, + 17, + -0.04217498789509575 + ], + [ + 31, + 27, + -0.24031267551163857 + ], + [ + 31, + 30, + -0.03958358827463807 + ] + ] +} \ No newline at end of file diff --git a/data/exp_gamma_field/EleutherAI--pythia-410m_gamma_field.json b/data/exp_gamma_field/EleutherAI--pythia-410m_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..f84d949e017bd06bbc83df5e719e663884d7f887 --- /dev/null +++ b/data/exp_gamma_field/EleutherAI--pythia-410m_gamma_field.json @@ -0,0 +1,990 @@ +{ + "model": "EleutherAI/pythia-410m", + "theta": 10000, + "T_train": 2048, + "n_layers": 24, + "n_heads": 16, + "d_head": 64, + "n_prompts": 100, + "seeds": [ + 42, + 123 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "gamma_matrix": [ + [ + 2.726055838530958, + 1.0073425125469087, + 1.5171373901813452, + 1.0496721091688748, + 1.153401153203036, + 1.2185284025798226, + 2.3128970965201576, + 1.7044832265660552, + 1.4323257858013425, + 1.8719545414687282, + 1.2523847435794127, + 1.371101419457763, + 1.3492153955868749, + 1.72834333950613, + 2.404071608762059, + 1.3236650900890818 + ], + [ + 1.297378069809053, + 1.582234496309131, + 1.596746801315529, + 1.9734770011415022, + 1.1276487941161182, + 1.447669137682753, + 1.6422296950529471, + 1.3820431953431414, + 1.4959523102504295, + 2.2752839613751026, + 1.5858534848144956, + 1.36273066116829, + 1.2697020698610506, + 2.0177032994080992, + 1.8115864132013562, + 1.1305211949420135 + ], + [ + 0.8196173023100031, + 1.1783455191452, + 0.9544239558116041, + 0.9369183028317772, + 0.9715142603678371, + 0.7557475675061813, + 1.6633402086845652, + 1.6721987126652975, + 1.1908014778097502, + 1.0956034986791419, + 1.0741903815619485, + 1.2633926760515437, + 0.8687762497292961, + 1.099367190218468, + 0.899192614610129, + 0.7987130423153362 + ], + [ + 0.7170105015679494, + 1.5260365445087043, + 1.0053254357226828, + 1.0802366445385387, + 1.1417989940173474, + 1.1574968230067977, + 1.3423560516739537, + 0.9463382896065721, + 0.9786501674673512, + 1.2075324156801537, + 1.2927320814308247, + 0.8448521870055951, + 1.3052138474897328, + 0.9371387708381207, + 1.2982688322796205, + 1.0700574738278124 + ], + [ + 1.392094302932288, + 0.9645918875072284, + 1.1792610803109804, + 1.1784614728803235, + 1.062873879583764, + 1.0999669516182669, + 0.9804838206366652, + 0.9589278065075242, + 1.016907777909968, + 0.1604038687221364, + 1.3361387131703166, + 0.9815858592691089, + 0.9828719109581309, + 1.2154670636358424, + 2.205273914323529, + 0.8258432082979329 + ], + [ + 0.9651203404078724, + 1.1561719154184984, + -0.2956786983168969, + 1.8938979406291279, + 1.2268134774072605, + 2.549008693276239, + 1.6662635569027766, + 1.7068454514915083, + 1.8594120355745658, + 1.1341109863149978, + 1.5407595837065966, + 2.6386759621155647, + 1.5356731386892757, + 0.7462855365194477, + 1.34879739352047, + 1.4387861075673918 + ], + [ + 0.17620027092690854, + 1.1231692958162676, + 1.0512022263652687, + 0.7567255030536726, + 0.9541289999347151, + 1.1994930579975132, + 1.0005008296745068, + 1.0353165908606663, + 1.0000465873495101, + 1.570499571596221, + 1.0248707257865297, + 0.7150471033044332, + 1.2668509713180391, + 2.1663999225936306, + 0.8231664624619577, + 0.6964639572005903 + ], + [ + 0.33048387612976005, + 0.7939306831475077, + 3.6287897081220346, + 0.8608629297681091, + 1.0650490272159712, + 0.9785113140453806, + 1.116334605109173, + 2.092811713510906, + 0.4019813833507316, + 2.282454487431723, + 2.364176491488095, + 1.381285433594133, + 0.8472478641502119, + 0.923081914663547, + 0.5948585085012981, + 0.6842744350920941 + ], + [ + 1.0623238371870845, + 2.5886120954510994, + 2.804119671959186, + 1.0575394939218865, + 1.069209637374175, + 0.8506996501791494, + 0.610858881533511, + 0.6660348404027873, + 2.5013657453439917, + 1.6053755112433714, + 1.0085574475908714, + 1.9348705597884868, + 1.692985637294813, + 0.9258568392888018, + 1.2930546265672618, + 1.150906759787014 + ], + [ + 0.6021704266067636, + 2.0150588640824343, + 1.0617856058539596, + 0.9215185567484375, + 1.1056078400809024, + 2.9738479585025024, + 1.082891444553355, + 0.9281933669357331, + 0.7819336816517967, + 0.8674036467388558, + 1.8873248160823022, + 1.4446485126802426, + 1.2383995009758695, + 2.673630340872261, + 1.1101243813312174, + 1.4309031775265089 + ], + [ + 0.7975440790492834, + 2.794889916309123, + 1.1698473203773316, + 1.1638937068061617, + 0.6679840140577139, + 2.6825624278916025, + 1.166739795788089, + 0.885593422778708, + 1.205842330758815, + 0.757012041235384, + 2.327030694439253, + 2.2534584372854805, + 0.8442033482583157, + 0.9643744768487991, + 3.6046868593616557, + 0.8322049173144582 + ], + [ + 1.786536803955796, + 0.8001148827978782, + 0.4778511816889097, + 0.8223927842537179, + 1.5423479016394768, + 1.2501647514374032, + 0.7122344855047319, + 0.9198624117554458, + 1.927462428706347, + 1.6785052756614964, + 2.0166351635855153, + 1.059894572347914, + 1.1784552633473504, + 0.8426850644521451, + 0.6261486505683279, + 0.9906463978374613 + ], + [ + 1.2683682079409697, + 0.9370978173013425, + 2.577769610082435, + 1.2508961425466483, + 2.453728963035811, + 2.7762118269384035, + 0.8598308825922686, + 1.055557575697965, + 0.831705995226792, + 0.8033111589015419, + 0.7407214477823857, + 0.904549820939016, + 0.6185655065175506, + 1.3341392776580219, + 1.193475265772751, + 1.111972799252095 + ], + [ + 3.1859029275464708, + 1.139341279518733, + 2.458041085736429, + 1.1233809534374999, + 0.9657534386279925, + 2.4278853575846218, + 1.2328498129530467, + 1.1269330287753936, + 1.553884964741587, + 1.0930416247100283, + 1.191756401602798, + 1.148551407186968, + 1.0372406710584552, + 1.4654362498008267, + 0.7769460932616479, + 1.7570174134385932 + ], + [ + 1.3171053248200615, + 2.516663691586333, + 0.7052852739363096, + 2.050093993744093, + 1.5024133941475661, + 0.9541823353527353, + 1.0173189480292462, + 1.2391274752704537, + 1.3193491104114348, + 1.6589276048646786, + 1.7431587957307921, + 1.0649337185769256, + 2.506198220226507, + 1.2127468812145608, + 1.4523665909424128, + 1.3649941022695737 + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ] + ], + "r2_matrix": [ + [ + 0.8553556623978569, + 0.9151584764732614, + 0.9146218589154905, + 0.9034939383916283, + 0.9429638548843251, + 0.6804901337861475, + 0.9300989981837473, + 0.8715954026070893, + 0.9654513196595924, + 0.9513055175307998, + 0.9700080157058338, + 0.9494267198806084, + 0.9388757374197798, + 0.9581247345885079, + 0.8595679490061209, + 0.9394888076233235 + ], + [ + 0.9422979354946033, + 0.5836727556901264, + 0.9167712953399493, + 0.9594980646213624, + 0.3015839660326185, + 0.872571237531854, + 0.8140466410344224, + 0.8837871839344865, + 0.7924117081151023, + 0.9030591561427164, + 0.8295020722977019, + 0.8603898882882517, + 0.9222291555656231, + 0.8900792898886258, + 0.9001967690201051, + 0.9329159942513225 + ], + [ + 0.9508269180953806, + 0.9066082190891342, + 0.9233563136624595, + 0.9501892900276827, + 0.8624603665013589, + 0.8295377431636466, + 0.9564920558896087, + 0.839692090265409, + 0.9529423038806661, + 0.9734357754739755, + 0.6841684457023187, + 0.9442741669291983, + 0.9503656019842832, + 0.8383744693603261, + 0.8119155142769677, + 0.8528294927510212 + ], + [ + 0.6843422938386585, + 0.5207892435992348, + 0.8991592753339761, + 0.8981741652411168, + 0.9097672696110717, + 0.8081074382558733, + 0.859529709332111, + 0.9135124133167347, + 0.7383474637464633, + 0.9419340381620196, + 0.7747603451010004, + 0.9452200133399571, + 0.896860377014807, + 0.8010653811648794, + 0.8488718309265368, + 0.8106289084817871 + ], + [ + 0.9336788122060558, + 0.9257294581399651, + 0.9829515505072082, + 0.9596837620121615, + 0.9417290766624602, + 0.9406890783729677, + 0.9895989546040265, + 0.9686008426052473, + 0.9744479633067755, + 0.01151402411739455, + 0.9345318997781761, + 0.9163348216346678, + 0.9480983912593797, + 0.8883678627599471, + 0.8205751184525989, + 0.8454348494426849 + ], + [ + 0.9028393843200576, + 0.9561057247409825, + 0.025688393536546394, + 0.9614620251010907, + 0.8471974038415836, + 0.9230092094296645, + 0.5750603233207603, + 0.9417002353764884, + 0.9371043751934375, + 0.6668936359298432, + 0.7865041811069587, + 0.8685201548128538, + 0.9029620613546436, + 0.7962024237059186, + 0.9234946892312419, + 0.97534835657549 + ], + [ + 0.03751067750177017, + 0.9367823670356412, + 0.9761776644882908, + 0.7977806380334413, + 0.7863800532550931, + 0.569227392017609, + 0.9768022949310724, + 0.8502715631062394, + 0.9252711145366096, + 0.800966882128767, + 0.9513765056600803, + 0.9019939287212475, + 0.9062108933337372, + 0.7306244579447507, + 0.680379707676612, + 0.8474330280931425 + ], + [ + 0.38695345849512863, + 0.8899031189770877, + 0.6710652155431005, + 0.3234637286859504, + 0.9106476919064922, + 0.832023335280792, + 0.9781873681458712, + 0.7872781612735754, + 0.12012728357784308, + 0.7902114908582675, + 0.7996450532157691, + 0.950746030941528, + 0.8965911110061715, + 0.8173658349657837, + 0.5292841287626893, + 0.9822276948652363 + ], + [ + 0.7620705019549343, + 0.9204950255094079, + 0.771785189127018, + 0.9273769984956002, + 0.9191870606186425, + 0.7664404064564413, + 0.8282805722901275, + 0.9695375624973194, + 0.9815400442749195, + 0.8565139381585263, + 0.9244161005549375, + 0.981235353722962, + 0.40510584452661924, + 0.9301337275289854, + 0.9559378146409198, + 0.932092644511894 + ], + [ + 0.8778364516021933, + 0.5784653114879296, + 0.8866784854743379, + 0.9509526072661065, + 0.9492566488511656, + 0.43638275606178745, + 0.9688588711843795, + 0.9845376888075145, + 0.08073629407914551, + 0.8873476937974808, + 0.9262014310982233, + 0.9792141993481943, + 0.9641023852574261, + 0.7673400256472435, + 0.9486698384380745, + 0.9688773266829123 + ], + [ + 0.9573448427832075, + 0.9607024713746369, + 0.8913144733354507, + 0.9490927520239534, + 0.9558514173911694, + 0.8760916282321431, + 0.9678859540666852, + 0.8819768392093981, + 0.8194040168193472, + 0.7072621652525086, + 0.9932369818092335, + 0.8375616762094629, + 0.9470682861204187, + 0.9246753901305256, + 0.9666302984795903, + 0.9081091741346589 + ], + [ + 0.9142225383268213, + 0.9424083871776079, + 0.6346966178594948, + 0.887223198787241, + 0.9604022476430693, + 0.9460320371046991, + 0.9208194031848084, + 0.9599596688781213, + 0.9765214213404044, + 0.9066101575555505, + 0.9359664938952013, + 0.9495711650986609, + 0.9401971882151526, + 0.689169917413692, + 0.7571861953958493, + 0.9582795336169989 + ], + [ + 0.8778879209009807, + 0.9847090221803562, + 0.8724960378409794, + 0.9155867062514729, + 0.9260556495108792, + 0.9715635511486135, + 0.9247757102431647, + 0.9367046459805644, + 0.8816770473397679, + 0.8878751015965283, + 0.9463839279175924, + 0.9388108203099077, + 0.8673305348330349, + 0.8379799621433284, + 0.9760519741282394, + 0.9842470319851269 + ], + [ + 0.865093443364172, + 0.9329523078187874, + 0.8144897664755089, + 0.9707825811266287, + 0.9459775183826917, + 0.8084792942663992, + 0.9512670044090751, + 0.9135153342643266, + 0.9842597222128104, + 0.9667822894005126, + 0.8777405405058174, + 0.9440123293641677, + 0.9732965193931189, + 0.990891571965329, + 0.8587562964376453, + 0.8873367379226497 + ], + [ + 0.9284725466037672, + 0.9733913012603146, + 0.8785564673088235, + 0.9097389293354321, + 0.9284447276770192, + 0.9577222718458529, + 0.8840540447048221, + 0.9859500431192065, + 0.9767702791277082, + 0.9102492875363224, + 0.879641046495415, + 0.9236788231739088, + 0.8599354707228055, + 0.9628876579427949, + 0.918758040994364, + 0.9897328160817639 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + ], + "summary": { + "gamma_mean": 1.328321630353653, + "gamma_std": 0.6022841062829021, + "gamma_min": -0.2956786983168969, + "gamma_max": 3.6287897081220346, + "gamma_range": 3.9244684064389315, + "gamma_global_fit": 0.9794385722098524, + "r2_global": 0.9921453725160345, + "gamma_mixture_pred": 1.328321630353653 + }, + "outlier_heads_high": [ + [ + 0, + 0, + 2.726055838530958 + ], + [ + 5, + 5, + 2.549008693276239 + ], + [ + 5, + 11, + 2.6386759621155647 + ], + [ + 7, + 2, + 3.6287897081220346 + ], + [ + 8, + 1, + 2.5886120954510994 + ], + [ + 8, + 2, + 2.804119671959186 + ], + [ + 9, + 5, + 2.9738479585025024 + ], + [ + 9, + 13, + 2.673630340872261 + ], + [ + 10, + 1, + 2.794889916309123 + ], + [ + 10, + 5, + 2.6825624278916025 + ], + [ + 10, + 14, + 3.6046868593616557 + ], + [ + 12, + 2, + 2.577769610082435 + ], + [ + 12, + 5, + 2.7762118269384035 + ], + [ + 13, + 0, + 3.1859029275464708 + ] + ], + "outlier_heads_low": [ + [ + 5, + 2, + -0.2956786983168969 + ] + ] +} \ No newline at end of file diff --git a/data/exp_gamma_field/EleutherAI--pythia-70m_gamma_field.json b/data/exp_gamma_field/EleutherAI--pythia-70m_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..bf691935f8912e74d0df04e11f4c1c199ecfed5c --- /dev/null +++ b/data/exp_gamma_field/EleutherAI--pythia-70m_gamma_field.json @@ -0,0 +1,181 @@ +{ + "model": "EleutherAI/pythia-70m", + "theta": 10000, + "T_train": 2048, + "n_layers": 6, + "n_heads": 8, + "d_head": 64, + "n_prompts": 80, + "seeds": [ + 42, + 123 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "gamma_matrix": [ + [ + 1.8956686484272032, + 3.1607076796410625, + 1.5092277905185227, + 1.0210524810235992, + 2.6697729658162896, + 2.240932252937181, + 1.4803242723530303, + 1.3678854949145218 + ], + [ + 1.2586911409518957, + 1.8122687350843756, + 0.9786895266365704, + 2.589333242086659, + 1.717841303033267, + 1.78869554492529, + 1.4044532884390961, + 1.7554952260931749 + ], + [ + 4.0457550638112805, + -0.07770150689082977, + 0.9714444650266668, + 2.571306583143918, + 1.395077069838456, + 1.4296950879817791, + 1.1774134620824253, + 0.7206641642289606 + ], + [ + NaN, + 1.2432695109055647, + 1.7455692649459635, + 1.0389092113357223, + 1.1166611212100948, + 0.2601625870112875, + NaN, + 1.088286409170004 + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ], + [ + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN, + NaN + ] + ], + "r2_matrix": [ + [ + 0.9383960605648823, + 0.9139226057527464, + 0.9652240633378439, + 0.8206546090602095, + 0.6616322324505621, + 0.7028864044477238, + 0.9409609617830748, + 0.4429954980892479 + ], + [ + 0.9073993062841719, + 0.7817046347608425, + 0.7292171915820077, + 0.8648359077897436, + 0.8405510602371034, + 0.7725355084897314, + 0.853363582608857, + 0.8436657005273044 + ], + [ + 0.9336359251200337, + 0.0006779054805270412, + 0.8617195475561965, + 0.7479993150898953, + 0.4146043954196934, + 0.9300853117312192, + 0.635240622716813, + 0.5288158074094154 + ], + [ + 0.0, + 0.16447268158343042, + 0.9477185945863832, + 0.9910871774672763, + 0.9718780075824707, + 0.014197730330358094, + 0.0, + 0.9841352431853003 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ], + [ + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0, + 0.0 + ] + ], + "summary": { + "gamma_mean": 1.5792517362227676, + "gamma_std": 0.8170482676877581, + "gamma_min": -0.07770150689082977, + "gamma_max": 4.0457550638112805, + "gamma_range": 4.1234565707021105, + "gamma_global_fit": 1.1218135130629137, + "r2_global": 0.98599467700819, + "gamma_mixture_pred": 1.5792517362227678 + }, + "outlier_heads_high": [ + [ + 2, + 0, + 4.0457550638112805 + ] + ], + "outlier_heads_low": [ + [ + 2, + 1, + -0.07770150689082977 + ] + ] +} \ No newline at end of file diff --git a/data/exp_gamma_field/info_horizon_results.json b/data/exp_gamma_field/info_horizon_results.json new file mode 100644 index 0000000000000000000000000000000000000000..a7d6c9e12c474cd54ba0d58a2d7e8768e6db3db9 --- /dev/null +++ b/data/exp_gamma_field/info_horizon_results.json @@ -0,0 +1,486 @@ +{ + "horizon_results": [ + { + "name": "pythia-14m", + "theta": 10000, + "T_train": 2048, + "gamma": 0.685, + "arch": "MHA", + "phase": "B", + "d_horizon_alpha1": 2643.7820305491095, + "d_horizon_alpha4": 10575.128122196438, + "alpha_opt_4xTtrain": 3.0985913003949634 + }, + { + "name": "pythia-31m", + "theta": 10000, + "T_train": 2048, + "gamma": 1.235, + "arch": "MHA", + "phase": "B", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "pythia-70m", + "theta": 10000, + "T_train": 2048, + "gamma": 0.748, + "arch": "MHA", + "phase": "B", + "d_horizon_alpha1": 2038.7975841991988, + "d_horizon_alpha4": 8155.190336796795, + "alpha_opt_4xTtrain": 4.018054594280709 + }, + { + "name": "pythia-160m", + "theta": 10000, + "T_train": 2048, + "gamma": 0.511, + "arch": "MHA", + "phase": "B", + "d_horizon_alpha1": 4576.773209797773, + "d_horizon_alpha4": 18307.092839191093, + "alpha_opt_4xTtrain": 1.789907348361264 + }, + { + "name": "pythia-410m", + "theta": 10000, + "T_train": 2048, + "gamma": 1.022, + "arch": "MHA", + "phase": "A", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "pythia-1b", + "theta": 10000, + "T_train": 2048, + "gamma": 0.931, + "arch": "MHA", + "phase": "A", + "d_horizon_alpha1": 505.3378343021414, + "d_horizon_alpha4": 2021.3513372085656, + "alpha_opt_4xTtrain": 16.210937404504737 + }, + { + "name": "pythia-1.4b", + "theta": 10000, + "T_train": 2048, + "gamma": 0.705, + "arch": "MHA", + "phase": "A", + "d_horizon_alpha1": 2446.879770674857, + "d_horizon_alpha4": 9787.519082699428, + "alpha_opt_4xTtrain": 3.3479372783978767 + }, + { + "name": "pythia-2.8b", + "theta": 10000, + "T_train": 2048, + "gamma": 0.674, + "arch": "MHA", + "phase": "A", + "d_horizon_alpha1": 2754.0837594601494, + "d_horizon_alpha4": 11016.335037840598, + "alpha_opt_4xTtrain": 2.974491960115906 + }, + { + "name": "gpt2-117m", + "theta": null, + "T_train": 1024, + "gamma": 1.023, + "arch": "AbsPE", + "phase": "B", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "gpt2-345m", + "theta": null, + "T_train": 1024, + "gamma": 0.784, + "arch": "AbsPE", + "phase": "A", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "gpt2-774m", + "theta": null, + "T_train": 1024, + "gamma": 0.753, + "arch": "AbsPE", + "phase": "A", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "gpt2-1.5b", + "theta": null, + "T_train": 1024, + "gamma": 1.01, + "arch": "AbsPE", + "phase": "B", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "Qwen2.5-0.5B", + "theta": 1000000, + "T_train": 8192, + "gamma": 1.028, + "arch": "GQA-MHA", + "phase": "B", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "Qwen2.5-3B", + "theta": 1000000, + "T_train": 8192, + "gamma": 0.772, + "arch": "GQA-n2", + "phase": "B", + "d_horizon_alpha1": 181964.2732624524, + "d_horizon_alpha4": 727857.0930498096, + "alpha_opt_4xTtrain": 0.1800793057477704 + }, + { + "name": "Qwen2.5-7B", + "theta": 1000000, + "T_train": 8192, + "gamma": 0.997, + "arch": "GQA", + "phase": "H", + "d_horizon_alpha1": 2124.507104215969, + "d_horizon_alpha4": 8498.028416863875, + "alpha_opt_4xTtrain": 15.423812862274593 + }, + { + "name": "gemma-2-9b", + "theta": null, + "T_train": 8192, + "gamma": 0.628, + "arch": "SWA", + "phase": "B", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "phi-3-mini", + "theta": null, + "T_train": 4096, + "gamma": 0.63, + "arch": "SWA", + "phase": "B", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "Llama-2-7b", + "theta": 10000, + "T_train": 4096, + "gamma": 1.026, + "arch": "MHA", + "phase": "B", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "Llama-3-8B", + "theta": 500000, + "T_train": 8192, + "gamma": 1.045, + "arch": "GQA", + "phase": "A", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "Mistral-7B", + "theta": 10000, + "T_train": 8192, + "gamma": 1.061, + "arch": "GQA", + "phase": "B", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "DeepSeek-7B", + "theta": 10000, + "T_train": 4096, + "gamma": 0.947, + "arch": "MHA", + "phase": "A", + "d_horizon_alpha1": 384.9682527261125, + "d_horizon_alpha4": 1539.87301090445, + "alpha_opt_4xTtrain": 42.55935361936579 + }, + { + "name": "phi-2", + "theta": 10000, + "T_train": 2048, + "gamma": 1.045, + "arch": "MHA", + "phase": "B", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "bloom-7b1", + "theta": null, + "T_train": 2048, + "gamma": 1.151, + "arch": "ALiBi", + "phase": "B", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "mamba-2.8b", + "theta": null, + "T_train": 2048, + "gamma": 0.703, + "arch": "SSM", + "phase": "A", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + }, + { + "name": "gpt-j-6B", + "theta": null, + "T_train": 2048, + "gamma": 0.897, + "arch": "AbsPE", + "phase": "A", + "d_horizon_alpha1": null, + "d_horizon_alpha4": null, + "alpha_opt_4xTtrain": null + } + ], + "phase_diagram": [ + { + "name": "pythia-14m", + "gamma_rand": 1.004, + "gamma_text": 0.685, + "delta_gamma": 0.31899999999999995, + "arch": "MHA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "pythia-31m", + "gamma_rand": 1.54, + "gamma_text": 1.235, + "delta_gamma": 0.30499999999999994, + "arch": "MHA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "pythia-70m", + "gamma_rand": 1.171, + "gamma_text": 0.748, + "delta_gamma": 0.42300000000000004, + "arch": "MHA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "pythia-160m", + "gamma_rand": 1.017, + "gamma_text": 0.511, + "delta_gamma": 0.5059999999999999, + "arch": "MHA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "pythia-410m", + "gamma_rand": 0.936, + "gamma_text": 1.022, + "delta_gamma": -0.08599999999999997, + "arch": "MHA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "pythia-1b", + "gamma_rand": 0.713, + "gamma_text": 0.931, + "delta_gamma": -0.21800000000000008, + "arch": "MHA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "pythia-1.4b", + "gamma_rand": 0.688, + "gamma_text": 0.705, + "delta_gamma": -0.017000000000000015, + "arch": "MHA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "pythia-2.8b", + "gamma_rand": 0.551, + "gamma_text": 0.674, + "delta_gamma": -0.123, + "arch": "MHA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "gpt2-117m", + "gamma_rand": 1.051, + "gamma_text": 1.023, + "delta_gamma": 0.028000000000000025, + "arch": "AbsPE", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "gpt2-345m", + "gamma_rand": 0.741, + "gamma_text": 0.784, + "delta_gamma": -0.04300000000000004, + "arch": "AbsPE", + "phase": "A", + "region": "post-IH" + }, + { + "name": "gpt2-774m", + "gamma_rand": 0.727, + "gamma_text": 0.753, + "delta_gamma": -0.026000000000000023, + "arch": "AbsPE", + "phase": "A", + "region": "post-IH" + }, + { + "name": "gpt2-1.5b", + "gamma_rand": 1.024, + "gamma_text": 1.01, + "delta_gamma": 0.014000000000000012, + "arch": "AbsPE", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "Qwen2.5-0.5B", + "gamma_rand": 0.919, + "gamma_text": 1.028, + "delta_gamma": -0.10899999999999999, + "arch": "GQA-MHA", + "phase": "B", + "region": "post-IH" + }, + { + "name": "Qwen2.5-3B", + "gamma_rand": 0.964, + "gamma_text": 0.772, + "delta_gamma": 0.19199999999999995, + "arch": "GQA-n2", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "Qwen2.5-7B", + "gamma_rand": 0.827, + "gamma_text": 0.997, + "delta_gamma": -0.17000000000000004, + "arch": "GQA", + "phase": "H", + "region": "post-IH" + }, + { + "name": "gemma-2-9b", + "gamma_rand": 1.135, + "gamma_text": 0.628, + "delta_gamma": 0.507, + "arch": "SWA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "phi-3-mini", + "gamma_rand": 1.037, + "gamma_text": 0.63, + "delta_gamma": 0.4069999999999999, + "arch": "SWA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "Llama-3-8B", + "gamma_rand": 0.759, + "gamma_text": 1.045, + "delta_gamma": -0.2859999999999999, + "arch": "GQA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "Mistral-7B", + "gamma_rand": 0.83, + "gamma_text": 1.061, + "delta_gamma": -0.23099999999999998, + "arch": "GQA", + "phase": "B", + "region": "post-IH" + }, + { + "name": "DeepSeek-7B", + "gamma_rand": 0.91, + "gamma_text": 0.947, + "delta_gamma": -0.03699999999999992, + "arch": "MHA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "phi-2", + "gamma_rand": 0.871, + "gamma_text": 1.045, + "delta_gamma": -0.17399999999999993, + "arch": "MHA", + "phase": "B", + "region": "post-IH" + }, + { + "name": "gpt-j-6B", + "gamma_rand": 0.835, + "gamma_text": 0.897, + "delta_gamma": -0.062000000000000055, + "arch": "AbsPE", + "phase": "A", + "region": "post-IH" + } + ], + "formula": { + "d_horizon": "theta * (1-gamma)*sqrt(2) / (1+gamma)", + "theta_design": "T_eval * sqrt(2) * (1+gamma) / (2*(1-gamma))", + "alpha_opt": "theta_design / theta_train", + "note": "d_horizon = T_eval when theta = theta_design. Inverse relationship.", + "validation": "pythia-70m: d_horizon(alpha=1)=2046, collapse at L=4096=2*d_horizon \u2713" + } +} \ No newline at end of file diff --git a/data/exp_gamma_field/phase_diagram_data.json b/data/exp_gamma_field/phase_diagram_data.json new file mode 100644 index 0000000000000000000000000000000000000000..71cdc0f205102c7de2058379c91da277c1a2f5a9 --- /dev/null +++ b/data/exp_gamma_field/phase_diagram_data.json @@ -0,0 +1,200 @@ +[ + { + "name": "pythia-14m", + "gamma_rand": 1.004, + "gamma_text": 0.685, + "delta_gamma": 0.31899999999999995, + "arch": "MHA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "pythia-31m", + "gamma_rand": 1.54, + "gamma_text": 1.235, + "delta_gamma": 0.30499999999999994, + "arch": "MHA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "pythia-70m", + "gamma_rand": 1.171, + "gamma_text": 0.748, + "delta_gamma": 0.42300000000000004, + "arch": "MHA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "pythia-160m", + "gamma_rand": 1.017, + "gamma_text": 0.511, + "delta_gamma": 0.5059999999999999, + "arch": "MHA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "pythia-410m", + "gamma_rand": 0.936, + "gamma_text": 1.022, + "delta_gamma": -0.08599999999999997, + "arch": "MHA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "pythia-1b", + "gamma_rand": 0.713, + "gamma_text": 0.931, + "delta_gamma": -0.21800000000000008, + "arch": "MHA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "pythia-1.4b", + "gamma_rand": 0.688, + "gamma_text": 0.705, + "delta_gamma": -0.017000000000000015, + "arch": "MHA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "pythia-2.8b", + "gamma_rand": 0.551, + "gamma_text": 0.674, + "delta_gamma": -0.123, + "arch": "MHA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "gpt2-117m", + "gamma_rand": 1.051, + "gamma_text": 1.023, + "delta_gamma": 0.028000000000000025, + "arch": "AbsPE", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "gpt2-345m", + "gamma_rand": 0.741, + "gamma_text": 0.784, + "delta_gamma": -0.04300000000000004, + "arch": "AbsPE", + "phase": "A", + "region": "post-IH" + }, + { + "name": "gpt2-774m", + "gamma_rand": 0.727, + "gamma_text": 0.753, + "delta_gamma": -0.026000000000000023, + "arch": "AbsPE", + "phase": "A", + "region": "post-IH" + }, + { + "name": "gpt2-1.5b", + "gamma_rand": 1.024, + "gamma_text": 1.01, + "delta_gamma": 0.014000000000000012, + "arch": "AbsPE", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "Qwen2.5-0.5B", + "gamma_rand": 0.919, + "gamma_text": 1.028, + "delta_gamma": -0.10899999999999999, + "arch": "GQA-MHA", + "phase": "B", + "region": "post-IH" + }, + { + "name": "Qwen2.5-3B", + "gamma_rand": 0.964, + "gamma_text": 0.772, + "delta_gamma": 0.19199999999999995, + "arch": "GQA-n2", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "Qwen2.5-7B", + "gamma_rand": 0.827, + "gamma_text": 0.997, + "delta_gamma": -0.17000000000000004, + "arch": "GQA", + "phase": "H", + "region": "post-IH" + }, + { + "name": "gemma-2-9b", + "gamma_rand": 1.135, + "gamma_text": 0.628, + "delta_gamma": 0.507, + "arch": "SWA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "phi-3-mini", + "gamma_rand": 1.037, + "gamma_text": 0.63, + "delta_gamma": 0.4069999999999999, + "arch": "SWA", + "phase": "B", + "region": "pre-IH" + }, + { + "name": "Llama-3-8B", + "gamma_rand": 0.759, + "gamma_text": 1.045, + "delta_gamma": -0.2859999999999999, + "arch": "GQA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "Mistral-7B", + "gamma_rand": 0.83, + "gamma_text": 1.061, + "delta_gamma": -0.23099999999999998, + "arch": "GQA", + "phase": "B", + "region": "post-IH" + }, + { + "name": "DeepSeek-7B", + "gamma_rand": 0.91, + "gamma_text": 0.947, + "delta_gamma": -0.03699999999999992, + "arch": "MHA", + "phase": "A", + "region": "post-IH" + }, + { + "name": "phi-2", + "gamma_rand": 0.871, + "gamma_text": 1.045, + "delta_gamma": -0.17399999999999993, + "arch": "MHA", + "phase": "B", + "region": "post-IH" + }, + { + "name": "gpt-j-6B", + "gamma_rand": 0.835, + "gamma_text": 0.897, + "delta_gamma": -0.062000000000000055, + "arch": "AbsPE", + "phase": "A", + "region": "post-IH" + } +] \ No newline at end of file diff --git a/data/exp_gamma_field/pythia-1b_step10000_gamma_field.json b/data/exp_gamma_field/pythia-1b_step10000_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..c59c24f2e8bebcda991158a0c13a0919421440e7 --- /dev/null +++ b/data/exp_gamma_field/pythia-1b_step10000_gamma_field.json @@ -0,0 +1,366 @@ +{ + "model": "EleutherAI/pythia-1b", + "revision": "step10000", + "n_prompts": 50, + "seeds": [ + 42 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "gamma_matrix": [ + [ + 1.0535772521023106, + 1.0667751051504801, + 1.0291188716063664, + 1.2367512866947261, + 2.447694729870426, + 1.3395037955773723, + 0.8702683440402289, + 1.2197279834611434 + ], + [ + 1.0922444667775442, + 2.2553909279157973, + 0.918754409573853, + 2.2700279759540942, + 1.1729283521316936, + 1.4193982123385158, + 1.8802988283146638, + 1.3488556269059855 + ], + [ + 0.649559196109082, + 2.1531120393679006, + 1.2412976283730237, + 2.2543678916953276, + 0.7154205957903726, + 0.745880520690942, + 0.6628058496237015, + 1.2509138972761809 + ], + [ + 1.1761205272949076, + 1.063119634376629, + 1.0697273079607132, + 0.6107809099107054, + 0.7012650763338552, + NaN, + 0.5575464276786057, + 0.43970665949815624 + ], + [ + 0.8700522405070729, + 1.0628096690400717, + 0.9321155308710658, + 0.5191210759441597, + 0.8077365017752346, + 1.1184292911192049, + 1.1016719342185608, + 0.9400695977917387 + ], + [ + 0.9986212721456817, + 1.0970056113239248, + 1.1210117979167307, + 0.7490672558415046, + 1.0182103557228188, + 0.6972640897839555, + 1.0755899379969651, + 0.9528739443597152 + ], + [ + 0.833171899249037, + 0.7942984569549497, + 1.1231746156882585, + -0.7479043308781903, + 1.1217480808392157, + 0.9722104872283797, + 1.1221798673662944, + 0.9392423958508276 + ], + [ + 1.0885889170529421, + 0.6889448531461106, + 1.0031141496435045, + 0.9451067702906971, + 2.266895115025411, + 0.723250653817733, + 1.0219227371317319, + 1.2339567378114487 + ], + [ + 1.199639245719769, + 1.1406884485897142, + 1.150414048391019, + 1.1706652351835758, + 1.1687680330871104, + 1.1064643863988421, + 1.008603862089979, + 1.0975711409748579 + ], + [ + 1.2865987595668549, + 0.9246923905276073, + 0.9906634124668907, + 1.233103661403579, + 0.8745311659681032, + 1.0581711391679973, + 1.2434647005366932, + 1.246421859106398 + ], + [ + 1.1360120764420685, + 1.0940906700000348, + 0.8083579625616636, + 0.8989222144113239, + 1.1217011516212447, + 0.9877786037647414, + 0.996162303066949, + 1.2336920843043948 + ], + [ + 0.9782028087678925, + 1.206437209902894, + 1.1632016790501154, + 0.9897344998878649, + 1.3184890609187423, + 1.2508834774518482, + 1.006931102054948, + 1.105259874769035 + ], + [ + 0.9635354002596613, + 1.0070007285186653, + 0.9515751123120041, + 1.1693805196541263, + 0.9291341241845463, + 1.0510261734630042, + 0.9815775512082217, + 0.9349935563220736 + ], + [ + 0.8873955157493836, + 1.198171648565688, + 1.0534077467848146, + 1.238867276582948, + 1.6161165498182788, + 1.18426322228356, + 1.050557485705403, + 0.8658316438738116 + ], + [ + 1.0570393100234596, + 1.3038544301257322, + 1.333967738298699, + 1.2597450541814499, + 1.0610159123183274, + 1.5543155977128775, + 0.7502580509760394, + 0.9614784754947194 + ], + [ + 1.3046327635940413, + 1.0225694926978455, + 1.1295109616192036, + 0.7604500729330591, + 0.8884892652100222, + 1.076654611508071, + 1.2024148640707282, + 1.340502003696035 + ] + ], + "r2_matrix": [ + [ + 0.8514849180921892, + 0.9905188293242001, + 0.8853550311095972, + 0.9269927235532658, + 0.9819946318039631, + 0.8976743402670575, + 0.9942936942918628, + 0.9139327939454028 + ], + [ + 0.5860883853976433, + 0.9246142009820438, + 0.9390663206962278, + 0.9195571113909501, + 0.681798861520587, + 0.909294878039529, + 0.8596910804172541, + 0.796812281366871 + ], + [ + 0.8799826135306359, + 0.8033809669201035, + 0.9544858416907436, + 0.9150131785556282, + 0.920845154916575, + 0.35867293768626163, + 0.5600774025753918, + 0.9730447748146022 + ], + [ + 0.8749970957098336, + 0.9866326514115943, + 0.8448887402733511, + 0.49530346784598756, + 0.8283940687918264, + 0.0, + 0.39831389548908325, + 0.19453678347523284 + ], + [ + 0.9108097805114606, + 0.678765280758395, + 0.8981009443406506, + 0.5599710671970134, + 0.777894195109916, + 0.961553725778152, + 0.950003668264799, + 0.6053647096004884 + ], + [ + 0.9507827497583345, + 0.6241994775160207, + 0.952487761003313, + 0.47233833714985274, + 0.8961547115703407, + 0.48351583645110974, + 0.927483915374746, + 0.9318318217332562 + ], + [ + 0.3702309510867664, + 0.9242004640653692, + 0.9829793063778558, + 0.07920493433524156, + 0.7145742707044849, + 0.4756505612709486, + 0.9783385147639888, + 0.9428925300628247 + ], + [ + 0.9147870000399525, + 0.7801614172425089, + 0.8530626166175264, + 0.5989736753212794, + 0.9745950420279902, + 0.3251623489754678, + 0.6436861322175277, + 0.9581143308735225 + ], + [ + 0.9281700889780538, + 0.9002478598704864, + 0.980917608150777, + 0.9482530752070197, + 0.7007731729876526, + 0.9583623529843771, + 0.9792492255074159, + 0.7012609985938347 + ], + [ + 0.9758072320602469, + 0.8308943696052081, + 0.6593089640647748, + 0.9408946023914386, + 0.9468024644251991, + 0.7956725607979531, + 0.7352474681546759, + 0.9637012595911632 + ], + [ + 0.8794689723755134, + 0.3714864470910111, + 0.3377550613648672, + 0.4319672929288082, + 0.9818201152105746, + 0.9615467091957285, + 0.9751976719495707, + 0.8547324805597768 + ], + [ + 0.8723364703558087, + 0.9593315931488118, + 0.9453776965350914, + 0.8758571565283815, + 0.9581312173115494, + 0.9152180896465365, + 0.8396390059279064, + 0.8970439734974647 + ], + [ + 0.9110459421782374, + 0.4866398609493813, + 0.9469226489952136, + 0.8769263783957273, + 0.5091329809954213, + 0.47115023535690415, + 0.9648645206782465, + 0.8101374236755563 + ], + [ + 0.6299423225087375, + 0.6664321050634164, + 0.9470314752781752, + 0.9833428883241853, + 0.9581035269916682, + 0.7469996318245313, + 0.9785154823778377, + 0.9296501815454414 + ], + [ + 0.8801697795413489, + 0.9124198759120065, + 0.98883793696114, + 0.9110561915154227, + 0.9324428437485205, + 0.9180295523598224, + 0.9305383642629217, + 0.8392649311446998 + ], + [ + 0.9007051174953675, + 0.9488851798102503, + 0.9228289064200071, + 0.8107782783350264, + 0.9164164732474314, + 0.853578943972904, + 0.9895588799395267, + 0.878827130800122 + ] + ], + "summary": { + "revision": "step10000", + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "gamma_global_fit": 0.9948340866027164, + "r2_global": 0.9815701512482831, + "mean_field": 1.0896893147312858, + "std_field": 0.373994827669732, + "min_field": -0.7479043308781903, + "max_field": 2.447694729870426, + "nan_pct": 0.78125 + } +} \ No newline at end of file diff --git a/data/exp_gamma_field/pythia-1b_step1000_gamma_field.json b/data/exp_gamma_field/pythia-1b_step1000_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..9bb6a2d692c02a31bb654f5261f6209ef6ac22f4 --- /dev/null +++ b/data/exp_gamma_field/pythia-1b_step1000_gamma_field.json @@ -0,0 +1,366 @@ +{ + "model": "EleutherAI/pythia-1b", + "revision": "step1000", + "n_prompts": 50, + "seeds": [ + 42 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "gamma_matrix": [ + [ + 0.7474749453476273, + 0.7076428704726426, + 0.6871219396321527, + 0.7490695330948428, + 0.684601950446864, + 0.7015262012761737, + 0.7184982133783883, + 0.6690929401114878 + ], + [ + 1.521125654254541, + 1.759716381538414, + 1.6280291298478036, + 0.93644910864332, + 1.0015535907560658, + 1.6174405174567579, + 0.9856246273781603, + 0.8050533637721918 + ], + [ + 1.1757348824182559, + 1.2288077666563781, + 1.3159811831241988, + 1.0328736853350704, + 1.310618313456728, + 1.2218838918237707, + 0.6554095902426164, + 0.8282597726097678 + ], + [ + 0.9737991680477757, + 1.3880654791375202, + 0.6236091522919587, + 1.016263715906122, + 1.1332721000733452, + -0.08560769769077914, + 0.5848153877522656, + 0.4636089552539435 + ], + [ + 0.9671704731293895, + 0.8302534222931083, + 0.7918803618680683, + 0.7822351977436505, + 0.6888944576357565, + 0.8150802656111888, + 0.9163767970622911, + 1.0824760108266918 + ], + [ + 1.134245670938439, + 0.6653615885795942, + 0.9906628389343443, + 0.8152998023427439, + 0.8238750096153729, + 0.6258611119619033, + 0.8305687569760293, + 0.5628854862100067 + ], + [ + 0.9920962352259711, + 1.0428085723108798, + 0.5701768277546173, + 0.2513041553673345, + 1.1624339374659118, + 0.9390940810644984, + 0.8226135834898253, + 0.38137388820226426 + ], + [ + 0.7542104849806089, + 0.7076466172995699, + 1.0470537444140737, + 0.714369392420496, + 1.2254261592493527, + 0.8336264910480928, + 0.7527012279660302, + 0.6594684594156787 + ], + [ + 0.8861205375757562, + 0.8250538723835337, + 0.7611717253332991, + 1.2739474087412923, + 0.9744344731589275, + 1.2264027897171956, + 0.9776580389766069, + 0.7154948697268448 + ], + [ + 1.0155461196415891, + 0.8703144975213101, + 0.9056973650695705, + 0.8749606483198274, + 1.07397649505365, + 1.165548026600471, + 1.0385512119822884, + 1.0860786265398257 + ], + [ + 1.1986314260762503, + 0.6683963025935327, + 0.4974806716637873, + 0.7038672034211542, + 0.7157317394698484, + 0.8750313135780138, + 0.7669087328719677, + 0.3479339040115841 + ], + [ + 0.6257615455597231, + 1.3974240907386082, + 0.8628967924798653, + 0.5743068409037255, + 1.0160967977251534, + 0.9766208827581453, + 1.3715946912797294, + 0.9803253384217062 + ], + [ + 1.0463684440018695, + 0.5423954452324053, + 1.181769599747747, + 0.8536989086433809, + 0.7608983986791136, + 0.5183642730190442, + 0.8727057802189325, + 1.0005077271421385 + ], + [ + 1.2668014447397238, + 0.7282809721557406, + 1.0283075326991669, + 1.7066665897429985, + 0.886416508435132, + 0.543697581900586, + 1.038703050419045, + 1.4394275874995932 + ], + [ + 0.672361675102875, + 0.8833417261252203, + 1.009224023791815, + 0.5596909871038663, + 1.1070957175458016, + 0.9397044661369496, + 1.0344748571304403, + 0.9586164378402141 + ], + [ + 0.8839826515181091, + 0.9162262791405725, + 0.6859186854822975, + 0.9172164973153478, + 0.9145036526072132, + 0.721303171881944, + 1.1535058680151822, + 0.6702854923980779 + ] + ], + "r2_matrix": [ + [ + 0.9866310118853248, + 0.9797838488030155, + 0.9715892288803827, + 0.9892560384769403, + 0.9746745356436342, + 0.9820807996426291, + 0.9756280348072407, + 0.9830602714633346 + ], + [ + 0.8214607277147027, + 0.9938733639195214, + 0.9106488047896941, + 0.5462687687873707, + 0.5571814851744886, + 0.9503459521049591, + 0.6893344804831492, + 0.9634459702531498 + ], + [ + 0.9023735536453941, + 0.835200225048234, + 0.9535057863961133, + 0.6171560551441133, + 0.9226593905603337, + 0.651038763499762, + 0.890045881275316, + 0.8886755524980502 + ], + [ + 0.8986883678307788, + 0.9421512131183333, + 0.7532160924090793, + 0.8044242614720855, + 0.8842210750781652, + 0.000997556766740404, + 0.3533795792370219, + 0.26086131577588834 + ], + [ + 0.9190286869824749, + 0.6689182855553304, + 0.9601957117431345, + 0.9688681659079926, + 0.7121522589881895, + 0.7886010237700112, + 0.9315615749199845, + 0.8666002022646729 + ], + [ + 0.8376280597997825, + 0.9600993465922204, + 0.9728400858366627, + 0.9072568325869359, + 0.9271709571957025, + 0.9729319368506172, + 0.5052146676445639, + 0.5211388318531307 + ], + [ + 0.987125631084528, + 0.9254818214849642, + 0.9309753954556876, + 0.02510703622052768, + 0.9770855261475591, + 0.9560055468861444, + 0.8788358263414506, + 0.42459782309941074 + ], + [ + 0.9784242474156188, + 0.9284115644759047, + 0.8281244148605709, + 0.8128816533228116, + 0.9099889801049095, + 0.9458226664528946, + 0.9264679141316254, + 0.9899492596071354 + ], + [ + 0.9786259456592116, + 0.9732834746303157, + 0.9764541957980446, + 0.9772885957621174, + 0.9534883138928151, + 0.9799308123865492, + 0.986873124628002, + 0.9219352336904811 + ], + [ + 0.9778633460410013, + 0.9299085542634604, + 0.9869319783384769, + 0.9646781626357724, + 0.9604297968724468, + 0.9513186509601176, + 0.975736699946382, + 0.9645565572148603 + ], + [ + 0.9204190464913323, + 0.8843334456273041, + 0.9330960851137274, + 0.9749065453140084, + 0.9042031012241001, + 0.9643522786516536, + 0.9217621084753951, + 0.9150280040897949 + ], + [ + 0.9256553239054037, + 0.9235414268486208, + 0.9068119868866527, + 0.787512795028121, + 0.8673720173789087, + 0.9623968357356842, + 0.9173204484849912, + 0.887491729254262 + ], + [ + 0.9000886799808188, + 0.8461523492530423, + 0.9283585433042723, + 0.9168296914587208, + 0.9846562520318137, + 0.7001685008143186, + 0.9810024155932173, + 0.6651312465479033 + ], + [ + 0.8675451835357733, + 0.8791535995901274, + 0.9675361485552776, + 0.9398247575782189, + 0.7612186096029059, + 0.9209731190297524, + 0.9518753987238884, + 0.9856455672202176 + ], + [ + 0.9157976943454142, + 0.8060663172068732, + 0.9223583009533804, + 0.5522131531379846, + 0.9171847060166813, + 0.7699492753791461, + 0.977888234451399, + 0.8719206165769973 + ], + [ + 0.7505022345305103, + 0.7292790488024073, + 0.7466804184870228, + 0.9716595221589694, + 0.9468524602158583, + 0.8052710205303817, + 0.9724337987834469, + 0.6037658330246016 + ] + ], + "summary": { + "revision": "step1000", + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "gamma_global_fit": 0.8240794658671543, + "r2_global": 0.9968657976189282, + "mean_field": 0.9035888627856054, + "std_field": 0.28631254979504756, + "min_field": -0.08560769769077914, + "max_field": 1.759716381538414, + "nan_pct": 0.0 + } +} \ No newline at end of file diff --git a/data/exp_gamma_field/pythia-1b_step143000_gamma_field.json b/data/exp_gamma_field/pythia-1b_step143000_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..bbec0a7a814471399ee0651287a4537686627326 --- /dev/null +++ b/data/exp_gamma_field/pythia-1b_step143000_gamma_field.json @@ -0,0 +1,366 @@ +{ + "model": "EleutherAI/pythia-1b", + "revision": "step143000", + "n_prompts": 50, + "seeds": [ + 42 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "gamma_matrix": [ + [ + 0.5348250828151762, + 1.0708883299449137, + 1.1351061870892565, + 1.333063504740924, + 2.0190859582035166, + 1.5349585009519817, + 1.084268146778366, + 1.0233469341677854 + ], + [ + 0.35381889213368833, + 1.7929090506060983, + 0.8160315992129422, + 2.748107361800009, + 2.550524160392296, + 0.9111529115248692, + 1.4313873882784063, + 1.0379184402872101 + ], + [ + 0.9540367092905566, + 1.7020735955509516, + 0.8505692493490865, + 1.2388148993302095, + 0.9552654112460723, + 0.5570576905792871, + 0.7929718504101192, + 1.1157689241588127 + ], + [ + 1.5014040799356778, + 0.9184847619884634, + 1.1023855603906432, + 0.7257810824814084, + 0.6418632659117768, + NaN, + 0.7985179135086734, + 0.7534785368050252 + ], + [ + 1.0644024740645308, + 1.7328491050924444, + 0.9896295837487361, + 0.619321636623625, + 0.8221127844471003, + 0.8152709459345252, + 0.9356883316813936, + 1.1332700316131126 + ], + [ + 0.8999169342581408, + 0.9597170076873092, + 0.7725655869001233, + 1.8846238421230292, + 0.9259598018835534, + 1.1650965009982914, + 0.942249439023053, + 0.9631569778446742 + ], + [ + 2.256247329968088, + 0.9542297077506069, + 1.1940011606151777, + 2.366130658465308, + 1.5668738503871489, + 2.0806293116304935, + 1.3385743764937061, + 1.0507013566304044 + ], + [ + 0.8485568330418788, + 0.48418423554237255, + 0.9842425303699778, + 1.9799226739420959, + 1.4532452932045372, + 3.1438843082858616, + 2.6464447525959134, + 2.0088629152001993 + ], + [ + 1.1664812500258785, + 1.3045501558719128, + 1.2959658455925114, + 1.853019721821141, + 2.148988884756393, + 1.479962839917099, + 1.4194337066014913, + 2.2222529646047535 + ], + [ + 1.1896787511561153, + 0.8303104193296372, + 2.023675754563125, + 1.599691980500783, + 1.066596307038827, + 1.022856981493219, + 2.122706184306042, + 1.0787746005452101 + ], + [ + 0.8941125649775106, + 3.2095074225052698, + 1.7023457370396058, + 1.7588648581708541, + 1.7541092609780147, + 0.4780410549748475, + 0.7843815783385546, + 1.7036940354082992 + ], + [ + 1.367339257492829, + 0.9601104984473551, + 0.8061088815604571, + 1.2129464803563186, + 1.068973100107751, + 0.7353315494186785, + 0.6442685493610625, + 0.6792260464031177 + ], + [ + 0.8193673669257183, + 1.8209526538547829, + 0.5647618492986348, + 0.6152328357317526, + 1.7339633784302997, + 1.4383928964220347, + 0.808778116009844, + 0.6077496843245813 + ], + [ + 2.2424870151529035, + 1.27420928305258, + 0.4066955061743055, + 0.8959204184726722, + 1.8420104966274762, + 1.2353172992401946, + 0.8621891076416586, + 0.9091017657179136 + ], + [ + 2.075814180049273, + 1.0286384840569873, + 1.454968838416083, + 1.4972259647093835, + 1.2574389856370305, + 1.7338916064388294, + 0.6521269922777942, + 1.6556592562383956 + ], + [ + 1.9158822833813258, + 1.2575540501190015, + 1.6001069537742196, + 0.7865672910896816, + 0.9229646944489904, + 1.6337476108329196, + 0.8165907789130951, + 1.854218479784366 + ] + ], + "r2_matrix": [ + [ + 0.5928742969983124, + 0.9584793442994842, + 0.9384016944734495, + 0.9974322519635989, + 0.9875272215715672, + 0.9888470244592322, + 0.943917303244883, + 0.8159323359610828 + ], + [ + 0.05436009581194701, + 0.9619222201447383, + 0.9644421511986804, + 0.6626673890894916, + 0.8148735056384022, + 0.9350486928177316, + 0.9044814119860172, + 0.8654267494411896 + ], + [ + 0.7973684570336931, + 0.9026598368683403, + 0.5342340349984429, + 0.7672881865822335, + 0.9826248681630032, + 0.26036325208011024, + 0.8752224160213037, + 0.9674313167189573 + ], + [ + 0.9075180771115435, + 0.9652159987982525, + 0.8351207109504383, + 0.6873181631035861, + 0.8721537594189905, + 0.0, + 0.4680189702850234, + 0.5888417731524351 + ], + [ + 0.9344871471712058, + 0.8804820919492488, + 0.9427487844893188, + 0.7247634323799296, + 0.8661768885880918, + 0.8573076309676851, + 0.9290719775440135, + 0.571809509790933 + ], + [ + 0.8339727665065068, + 0.7489969999422101, + 0.9457260944470675, + 0.8454246711144584, + 0.9721405667905401, + 0.4170992611745872, + 0.975016555076026, + 0.8256363928380357 + ], + [ + 0.7432816832921514, + 0.9422789874952483, + 0.97721664559652, + 0.9682848066709983, + 0.9713783772225681, + 0.8077184469174168, + 0.9168353160029549, + 0.9820021843878165 + ], + [ + 0.8889717492206707, + 0.8649136133907973, + 0.899470813851722, + 0.5303225033293408, + 0.9574297543736562, + 0.7820242115664505, + 0.7738464808224101, + 0.9611740803961277 + ], + [ + 0.9510798238023935, + 0.99062178785739, + 0.9940700042301376, + 0.9822923352703082, + 0.944053187491981, + 0.9750344051063757, + 0.9707640298239228, + 0.9703723198858605 + ], + [ + 0.9488123102914493, + 0.8619942034875138, + 0.9106530799401847, + 0.9937279545236388, + 0.9490330597397798, + 0.963993765678679, + 0.9437847829274031, + 0.9710085122500995 + ], + [ + 0.9506557471961338, + 0.7227599790678276, + 0.8914803055340264, + 0.8723189446447223, + 0.8406435473463627, + 0.9114283791627503, + 0.8927261064127268, + 0.9605606637447013 + ], + [ + 0.9769066750568655, + 0.9742366865731867, + 0.8682204642392837, + 0.989956106975503, + 0.9079453115977479, + 0.8734185372245925, + 0.7693763612351503, + 0.952684002853343 + ], + [ + 0.9412656116250243, + 0.955278295855092, + 0.9464493505012834, + 0.7942110298138293, + 0.978960937743435, + 0.8485226540572326, + 0.7840293979989684, + 0.7413143293842829 + ], + [ + 0.9121068568451776, + 0.8407487680405703, + 0.7874668915942769, + 0.9722283773439274, + 0.9306798473105112, + 0.9066956171265006, + 0.8960513205663071, + 0.8406938451221662 + ], + [ + 0.847080919198389, + 0.8992786605785619, + 0.9386273141282551, + 0.9707871088215639, + 0.9809804743866585, + 0.8805491871728743, + 0.9437735255902724, + 0.9667172773765855 + ], + [ + 0.9469820480258867, + 0.9500289075974633, + 0.9699905847417404, + 0.8466547099208634, + 0.9217229687788151, + 0.9451127850024401, + 0.8518434603930631, + 0.9236951387165351 + ] + ], + "summary": { + "revision": "step143000", + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "gamma_global_fit": 0.9847988928878217, + "r2_global": 0.9917497270124693, + "mean_field": 1.2816792252348113, + "std_field": 0.5683879149031182, + "min_field": 0.35381889213368833, + "max_field": 3.2095074225052698, + "nan_pct": 0.78125 + } +} \ No newline at end of file diff --git a/data/exp_gamma_field/pythia-1b_step1_gamma_field.json b/data/exp_gamma_field/pythia-1b_step1_gamma_field.json new file mode 100644 index 0000000000000000000000000000000000000000..a77a7ca587f3ce4b5c280c710125083e975349e4 --- /dev/null +++ b/data/exp_gamma_field/pythia-1b_step1_gamma_field.json @@ -0,0 +1,366 @@ +{ + "model": "EleutherAI/pythia-1b", + "revision": "step1", + "n_prompts": 50, + "seeds": [ + 42 + ], + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "gamma_matrix": [ + [ + 0.7356676138200544, + 0.7182849005168019, + 0.6990462202864575, + 0.7369992606389184, + 0.6958450690107368, + 0.7073459260273769, + 0.718644619648709, + 0.6824232686131939 + ], + [ + 0.7298819257885184, + 0.7170127136836519, + 0.6893725319833389, + 0.7194620911618821, + 0.7153753009385392, + 0.7209508307783485, + 0.7041711942997437, + 0.6990811221852565 + ], + [ + 0.7156256852754442, + 0.7010476350360824, + 0.71152325029764, + 0.676822172392102, + 0.7175463136227956, + 0.7390414903751871, + 0.7211225078073868, + 0.6776404700687246 + ], + [ + 0.7410063333220231, + 0.7033569574560432, + 0.6693010401186135, + 0.6944838474776446, + 0.7028835813488803, + 0.7225379543097976, + 0.7209522127695565, + 0.6971499424028684 + ], + [ + 0.7268000878131934, + 0.74562542192594, + 0.7528204573850166, + 0.7319337095755288, + 0.7141514549623373, + 0.7009979839014421, + 0.695416924587426, + 0.6975987997659567 + ], + [ + 0.6760121559339306, + 0.7467906423085787, + 0.718191605825076, + 0.6992240817778507, + 0.7342090604555283, + 0.7280150486598401, + 0.6901259734191653, + 0.6814599411115455 + ], + [ + 0.6831790884258581, + 0.7302678148679861, + 0.6934919280925809, + 0.7724441666908572, + 0.7438075347062876, + 0.6711560675686535, + 0.736935267342129, + 0.6771376546722496 + ], + [ + 0.7462288756873612, + 0.686933394957827, + 0.739389004170183, + 0.7104730029713693, + 0.7515687677123575, + 0.7364797108318091, + 0.6856527958070746, + 0.7007133471310074 + ], + [ + 0.7234981609560177, + 0.7308834519327602, + 0.7672718692278154, + 0.7496461310690031, + 0.6807121650506858, + 0.718641555968225, + 0.7038362979209434, + 0.7439016473202699 + ], + [ + 0.6824831089712017, + 0.7118667837864332, + 0.6967468023794117, + 0.7212776662607161, + 0.7413264197612156, + 0.7236325188839005, + 0.6996441038027502, + 0.7311694548099028 + ], + [ + 0.707969527756881, + 0.7024335366603164, + 0.6919358350213618, + 0.7342538362596267, + 0.6939998635717188, + 0.6750752817998632, + 0.7665057024908206, + 0.6898203469682673 + ], + [ + 0.7435240725264246, + 0.7411163565651523, + 0.764308322545504, + 0.6883835649735719, + 0.722119586968884, + 0.68383803597046, + 0.6751860721128998, + 0.6792225563443134 + ], + [ + 0.6817792929124327, + 0.6697522921503578, + 0.7523797546111614, + 0.6677491697781692, + 0.7067880826713638, + 0.7312947821201873, + 0.7314750337316769, + 0.733630694080852 + ], + [ + 0.7253235244360413, + 0.7170060435886494, + 0.706345653511234, + 0.6454507235246979, + 0.6958128715619507, + 0.7086929802486966, + 0.71787163558571, + 0.6979135847469947 + ], + [ + 0.7554098867163326, + 0.7228496694237304, + 0.73974417456992, + 0.729629349305403, + 0.7301664717175057, + 0.7198138800337706, + 0.6828213135515191, + 0.7222827840224174 + ], + [ + 0.6727499553879852, + 0.7176044010691268, + 0.7031443334815181, + 0.697241091773457, + 0.7279841039393287, + 0.7055170529064014, + 0.6592013264174249, + 0.7014246698969664 + ] + ], + "r2_matrix": [ + [ + 0.9784624687760999, + 0.9817416670770044, + 0.9771947436620835, + 0.9813317183837769, + 0.9784340790365712, + 0.9825148443276847, + 0.9793871095080977, + 0.9835214722240662 + ], + [ + 0.977881212134942, + 0.98494401476165, + 0.984964024082039, + 0.9835009284111523, + 0.9784487901654733, + 0.9710984282464503, + 0.9895027351419875, + 0.9736840257827989 + ], + [ + 0.9888401076096748, + 0.963415627793645, + 0.9874879303703267, + 0.9910705026016505, + 0.9825870096025082, + 0.9895789092975801, + 0.9849372102107149, + 0.9769804492745293 + ], + [ + 0.988385545815911, + 0.9829741078398535, + 0.9787586782142531, + 0.9807184437170954, + 0.9911232389036609, + 0.9819909167368273, + 0.9795908541676924, + 0.9923509837403547 + ], + [ + 0.9762618948148659, + 0.982779731687491, + 0.9797490822207022, + 0.9747640242454938, + 0.9871608147792861, + 0.9774787605056502, + 0.9796010061442235, + 0.9786457563466086 + ], + [ + 0.9835725225648696, + 0.9712907119337534, + 0.9766507088762997, + 0.9910316788807593, + 0.9705080535523912, + 0.9960960370857843, + 0.9681214748927521, + 0.966907026374558 + ], + [ + 0.9772119664629153, + 0.9810386383841081, + 0.9902986804246974, + 0.9739081529848173, + 0.9911039870898136, + 0.9782647137596343, + 0.9735106485509043, + 0.9693362131396154 + ], + [ + 0.978515263862252, + 0.9898108653349923, + 0.9809501573493987, + 0.9756255361113609, + 0.9675113218312126, + 0.9834683275390015, + 0.9856055871006921, + 0.9721297138258252 + ], + [ + 0.9770830394287178, + 0.9769243678800427, + 0.9796963819470681, + 0.9896313632805033, + 0.9714431172170667, + 0.9885782950152993, + 0.9845316887385651, + 0.985567322452426 + ], + [ + 0.9713554189117333, + 0.9918177118628575, + 0.9789169231104208, + 0.9719268877359998, + 0.9886337595120389, + 0.976531816049118, + 0.9423411420456581, + 0.9910388080445414 + ], + [ + 0.9749868961703887, + 0.9854786475713643, + 0.9878455989375072, + 0.9718623777480544, + 0.9810366105200087, + 0.9598192919138984, + 0.9532783027867647, + 0.9773618065393721 + ], + [ + 0.9693948879600245, + 0.9855011669922573, + 0.9872370716427206, + 0.9762511072330312, + 0.967196795519408, + 0.9642141586194029, + 0.9740293111442169, + 0.977176843910043 + ], + [ + 0.9750813750570323, + 0.9536040034222635, + 0.9879043626454804, + 0.9681328432732553, + 0.9794329460300671, + 0.9854346327235949, + 0.994712954623444, + 0.9794835313428276 + ], + [ + 0.9677194590497683, + 0.9749129545248865, + 0.9913463432694439, + 0.987052535846629, + 0.9465304874364078, + 0.976348074259176, + 0.9896914667859188, + 0.9816477660225873 + ], + [ + 0.9691491141897768, + 0.973384772913421, + 0.9875631651058792, + 0.982155607362909, + 0.982704970392314, + 0.9876702272188782, + 0.9825110301684823, + 0.9848528647381749 + ], + [ + 0.9754513865482073, + 0.9781220915519563, + 0.9894749413017307, + 0.9880802833962061, + 0.952099405038532, + 0.9939917444530383, + 0.9716916055801376, + 0.9900449592583987 + ] + ], + "summary": { + "revision": "step1", + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "gamma_global_fit": 0.7119770200889792, + "r2_global": 0.9833260935732103, + "mean_field": 0.7122649919891919, + "std_field": 0.02522550272222904, + "min_field": 0.6454507235246979, + "max_field": 0.7724441666908572, + "nan_pct": 0.0 + } +} \ No newline at end of file diff --git a/data/exp_gamma_intervention/results.json b/data/exp_gamma_intervention/results.json new file mode 100644 index 0000000000000000000000000000000000000000..cefbb54cafb47582504685cffdda41553b2f183a --- /dev/null +++ b/data/exp_gamma_intervention/results.json @@ -0,0 +1,53 @@ +{ + "model": "EleutherAI/pythia-1b", + "n_chunks": 50, + "seq_len": 1024, + "results": { + "eps_0.0": { + "epsilon": 0.0, + "ppl": 15.714571287245102, + "mean_nll": 2.7545883893966674, + "std_nll": 0.2679922705033423, + "n_chunks": 50, + "elapsed_sec": 7.0285985469818115, + "mean_spread_norm": null + }, + "eps_0.1": { + "epsilon": 0.1, + "ppl": 15.720479734077006, + "mean_nll": 2.754964303970337, + "std_nll": 0.26784558311564427, + "n_chunks": 50, + "elapsed_sec": 5.397161245346069, + "mean_spread_norm": 0.9999998443573713 + }, + "eps_0.3": { + "epsilon": 0.3, + "ppl": 15.743350115995264, + "mean_nll": 2.756418061256409, + "std_nll": 0.267586963460587, + "n_chunks": 50, + "elapsed_sec": 5.067129611968994, + "mean_spread_norm": 0.9999998462945223 + }, + "eps_0.5": { + "epsilon": 0.5, + "ppl": 15.779192556951937, + "mean_nll": 2.758692145347595, + "std_nll": 0.2672898058897529, + "n_chunks": 50, + "elapsed_sec": 5.041538238525391, + "mean_spread_norm": 0.9999998477101326 + }, + "eps_1.0": { + "epsilon": 1.0, + "ppl": 15.912686739617495, + "mean_nll": 2.76711669921875, + "std_nll": 0.2665913433348051, + "n_chunks": 50, + "elapsed_sec": 4.999021053314209, + "mean_spread_norm": 0.9999998519569635 + } + }, + "baseline_ppl": 15.714571287245102 +} \ No newline at end of file diff --git a/data/exp_kv_decay/EleutherAI--pythia-1.4b_kv_decay_seq4096.json b/data/exp_kv_decay/EleutherAI--pythia-1.4b_kv_decay_seq4096.json new file mode 100644 index 0000000000000000000000000000000000000000..9893d70e46507ce5c0a933fdf4ab18c35744b367 --- /dev/null +++ b/data/exp_kv_decay/EleutherAI--pythia-1.4b_kv_decay_seq4096.json @@ -0,0 +1,40 @@ +{ + "model": "EleutherAI/pythia-1.4b", + "seq_len": 4096, + "n_chunks": 30, + "theta": 10000.0, + "T_train": 2048, + "gamma_used": 0.705, + "gamma_pade_at_T_train": 0.7470064429851826, + "gamma_pade_at_seq_len": 0.5508312818257147, + "d_horizon": 2446.879770674857, + "D_f_f0_9": 2960, + "f_retain": 0.9, + "modes": { + "baseline": { + "ppl": 53.65674917940454, + "nll": 3.9826072613398233, + "std": 0.2010077981665634, + "delta_ppl": 0.0 + }, + "hard_cutoff": { + "ppl": 31.65464082972793, + "nll": 3.4548847675323486, + "std": 0.206414518381513, + "delta_ppl": -22.00210834967661 + }, + "soft_decay": { + "ppl": 25.16093004937277, + "nll": 3.2252923965454103, + "std": 0.19782993796895576, + "delta_ppl": -28.495819130031773 + }, + "hard_df": { + "ppl": 48.386144400553235, + "nll": 3.879213500022888, + "std": 0.18416963648784698, + "delta_ppl": -5.2706047788513075 + } + }, + "verdict": "SOFT_DECAY_WINS \u2014 idea 4 v25 confirmed" +} \ No newline at end of file diff --git a/data/exp_kv_decay/EleutherAI--pythia-1b_kv_decay_seq4096.json b/data/exp_kv_decay/EleutherAI--pythia-1b_kv_decay_seq4096.json new file mode 100644 index 0000000000000000000000000000000000000000..cae44ff12d9525e6f8e3b832f2de429623b7abb5 --- /dev/null +++ b/data/exp_kv_decay/EleutherAI--pythia-1b_kv_decay_seq4096.json @@ -0,0 +1,40 @@ +{ + "model": "EleutherAI/pythia-1b", + "seq_len": 4096, + "n_chunks": 30, + "theta": 10000.0, + "T_train": 2048, + "gamma_used": 0.931, + "gamma_pade_at_T_train": 0.7470064429851826, + "gamma_pade_at_seq_len": 0.5508312818257147, + "d_horizon": 505.3378343021414, + "D_f_f0_9": 2145, + "f_retain": 0.9, + "modes": { + "baseline": { + "ppl": 79.08715824739956, + "nll": 4.370550513267517, + "std": 0.25347707687263066, + "delta_ppl": 0.0 + }, + "hard_cutoff": { + "ppl": 25.261271320317242, + "nll": 3.229272445042928, + "std": 0.20028306722355718, + "delta_ppl": -53.825886927082315 + }, + "soft_decay": { + "ppl": 26.720916594771683, + "nll": 3.2854466517766316, + "std": 0.2015519103709852, + "delta_ppl": -52.36624165262788 + }, + "hard_df": { + "ppl": 19.954902748726713, + "nll": 2.9934748649597167, + "std": 0.22525463122726258, + "delta_ppl": -59.13225549867285 + } + }, + "verdict": "SOFT_DECAY_LOSES \u2014 idea 4 v25 refuted" +} \ No newline at end of file diff --git a/data/exp_kv_decay/EleutherAI--pythia-2.8b_kv_decay_seq4096.json b/data/exp_kv_decay/EleutherAI--pythia-2.8b_kv_decay_seq4096.json new file mode 100644 index 0000000000000000000000000000000000000000..9cd734ffeadba9cc1ea3cb05ca92ddb8d0722d93 --- /dev/null +++ b/data/exp_kv_decay/EleutherAI--pythia-2.8b_kv_decay_seq4096.json @@ -0,0 +1,40 @@ +{ + "model": "EleutherAI/pythia-2.8b", + "seq_len": 4096, + "n_chunks": 30, + "theta": 10000.0, + "T_train": 2048, + "gamma_used": 0.674, + "gamma_pade_at_T_train": 0.7470064429851826, + "gamma_pade_at_seq_len": 0.5508312818257147, + "d_horizon": 2754.0837594601494, + "D_f_f0_9": 3033, + "f_retain": 0.9, + "modes": { + "baseline": { + "ppl": 113.53278675197002, + "nll": 4.732091665267944, + "std": 0.21647391872688937, + "delta_ppl": 0.0 + }, + "hard_cutoff": { + "ppl": 120.23211487669701, + "nll": 4.789424165089925, + "std": 0.2085787817992625, + "delta_ppl": 6.699328124726989 + }, + "soft_decay": { + "ppl": 118.84341363067055, + "nll": 4.777806774775187, + "std": 0.20823771624144086, + "delta_ppl": 5.310626878700532 + }, + "hard_df": { + "ppl": 120.91026393710165, + "nll": 4.7950486501057945, + "std": 0.21036379187805948, + "delta_ppl": 7.3774771851316245 + } + }, + "verdict": "SOFT_DECAY_WINS \u2014 idea 4 v25 confirmed" +} \ No newline at end of file diff --git a/data/exp_mobius_cluster/mobius_cluster_results.json b/data/exp_mobius_cluster/mobius_cluster_results.json new file mode 100644 index 0000000000000000000000000000000000000000..22df981726d0882e8e9560ab3f89debb2d0f3ec8 --- /dev/null +++ b/data/exp_mobius_cluster/mobius_cluster_results.json @@ -0,0 +1,289 @@ +{ + "z_star": 0.5615528128088303, + "gamma_star": 0.5615528128088303, + "rows": [ + { + "label": "pythia-160m", + "family": "pythia", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 0.511, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.05055281280883028, + "fp_dist_composite": 0.6876755210243063 + }, + { + "label": "pythia-2.8b", + "family": "pythia", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 0.674, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.11244718719116975, + "fp_dist_composite": 0.694972233431216 + }, + { + "label": "pythia-14m", + "family": "pythia", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 0.685, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.12344718719116976, + "fp_dist_composite": 0.6968365973157395 + }, + { + "label": "pythia-1.4b", + "family": "pythia", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 0.705, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.14344718719116967, + "fp_dist_composite": 0.7006562144491584 + }, + { + "label": "pythia-70m", + "family": "pythia", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 0.748, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.1864471871911697, + "fp_dist_composite": 0.7107071048924906 + }, + { + "label": "pythia-1b", + "family": "pythia", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 0.931, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.36944718719116976, + "fp_dist_composite": 0.7789950317278241 + }, + { + "label": "DeepSeek-7B", + "family": "deepseek", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 0.947, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.38544718719116966, + "fp_dist_composite": 0.7867093297061826 + }, + { + "label": "pythia-410m", + "family": "pythia", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 1.022, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.4604471871911697, + "fp_dist_composite": 0.826047000796823 + }, + { + "label": "phi-2", + "family": "phi", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 1.045, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.48344718719116964, + "fp_dist_composite": 0.8390847502703289 + }, + { + "label": "Mistral-7B", + "family": "mistral", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 1.061, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.49944718719116965, + "fp_dist_composite": 0.8484041066180301 + }, + { + "label": "pythia-31m", + "family": "pythia", + "T_eval": 2000, + "theta": 10000, + "z": 0.282842712474619, + "gamma_obs": 1.235, + "gamma_pade_pred": 0.7522013138014093, + "log_z_dist_from_z_star": 0.6858148695771815, + "gamma_dist_from_g_star": 0.6734471871911698, + "fp_dist_composite": 0.9611832027604649 + }, + { + "label": "gpt-j-6B", + "family": "gptj", + "T_eval": 1000, + "theta": 10000, + "z": 0.1414213562373095, + "gamma_obs": 0.897, + "gamma_pade_pred": 0.8679182349373773, + "log_z_dist_from_z_star": 1.3789620501371267, + "gamma_dist_from_g_star": 0.3354471871911697, + "fp_dist_composite": 1.4191762227126183 + }, + { + "label": "SmolLM2-135M", + "family": "smollm2", + "T_eval": 2000, + "theta": 100000, + "z": 0.0282842712474619, + "gamma_obs": 0.748, + "gamma_pade_pred": 0.9721101507826946, + "log_z_dist_from_z_star": 2.988399962571227, + "gamma_dist_from_g_star": 0.1864471871911697, + "fp_dist_composite": 2.994210562052577 + }, + { + "label": "SmolLM2-1.7B", + "family": "smollm2", + "T_eval": 2000, + "theta": 100000, + "z": 0.0282842712474619, + "gamma_obs": 0.846, + "gamma_pade_pred": 0.9721101507826946, + "log_z_dist_from_z_star": 2.988399962571227, + "gamma_dist_from_g_star": 0.2844471871911697, + "fp_dist_composite": 3.001906817107533 + }, + { + "label": "SmolLM2-360M", + "family": "smollm2", + "T_eval": 2000, + "theta": 100000, + "z": 0.0282842712474619, + "gamma_obs": 0.969, + "gamma_pade_pred": 0.9721101507826946, + "log_z_dist_from_z_star": 2.988399962571227, + "gamma_dist_from_g_star": 0.4074471871911697, + "fp_dist_composite": 3.0160483329425785 + }, + { + "label": "Llama-3.2-3B", + "family": "llama", + "T_eval": 2000, + "theta": 500000, + "z": 0.00565685424949238, + "gamma_obs": 0.594, + "gamma_pade_pred": 0.9943591006233126, + "log_z_dist_from_z_star": 4.597837875005328, + "gamma_dist_from_g_star": 0.03244718719116968, + "fp_dist_composite": 4.597952364345473 + }, + { + "label": "Llama-3-8B", + "family": "llama", + "T_eval": 2000, + "theta": 500000, + "z": 0.00565685424949238, + "gamma_obs": 1.045, + "gamma_pade_pred": 0.9943591006233126, + "log_z_dist_from_z_star": 4.597837875005328, + "gamma_dist_from_g_star": 0.48344718719116964, + "fp_dist_composite": 4.623184433660047 + }, + { + "label": "Qwen2.5-3B", + "family": "qwen", + "T_eval": 2000, + "theta": 1000000, + "z": 0.00282842712474619, + "gamma_obs": 0.799, + "gamma_pade_pred": 0.9971755672263881, + "log_z_dist_from_z_star": 5.290985055565272, + "gamma_dist_from_g_star": 0.23744718719116975, + "fp_dist_composite": 5.296310416216184 + }, + { + "label": "Qwen2.5-7B", + "family": "qwen", + "T_eval": 2000, + "theta": 1000000, + "z": 0.00282842712474619, + "gamma_obs": 0.997, + "gamma_pade_pred": 0.9971755672263881, + "log_z_dist_from_z_star": 5.290985055565272, + "gamma_dist_from_g_star": 0.4354471871911697, + "fp_dist_composite": 5.308873431439833 + }, + { + "label": "Qwen2.5-0.5B", + "family": "qwen", + "T_eval": 2000, + "theta": 1000000, + "z": 0.00282842712474619, + "gamma_obs": 1.028, + "gamma_pade_pred": 0.9971755672263881, + "log_z_dist_from_z_star": 5.290985055565272, + "gamma_dist_from_g_star": 0.46644718719116973, + "fp_dist_composite": 5.311505985749578 + }, + { + "label": "Qwen2.5-1.5B", + "family": "qwen", + "T_eval": 2000, + "theta": 1000000, + "z": 0.00282842712474619, + "gamma_obs": 1.063, + "gamma_pade_pred": 0.9971755672263881, + "log_z_dist_from_z_star": 5.290985055565272, + "gamma_dist_from_g_star": 0.5014471871911697, + "fp_dist_composite": 5.314693983641672 + }, + { + "label": "Mistral-Nemo", + "family": "mistral", + "T_eval": 500, + "theta": 1000000, + "z": 0.0007071067811865475, + "gamma_obs": 0.541, + "gamma_pade_pred": 0.9992931431304565, + "log_z_dist_from_z_star": 6.677279416685163, + "gamma_dist_from_g_star": 0.020552812808830256, + "fp_dist_composite": 6.677311047615029 + }, + { + "label": "Yi-6B", + "family": "yi", + "T_eval": 2000, + "theta": 5000000, + "z": 0.000565685424949238, + "gamma_obs": 0.842, + "gamma_pade_pred": 0.9994344745298088, + "log_z_dist_from_z_star": 6.900422967999373, + "gamma_dist_from_g_star": 0.2804471871911697, + "fp_dist_composite": 6.906119587879775 + } + ], + "n_within_radius_0_5": 0, + "n_within_radius_1_0": 11, + "n_total": 23, + "null_fraction_within_0_5": 0.06544984694978735, + "observed_fraction_within_0_5": 0.0, + "enrichment_vs_null": 0.0, + "verdict": "Y UNSUPPORTED (no enrichment vs uniform)" +} \ No newline at end of file diff --git a/data/exp_multi_controller/results.json b/data/exp_multi_controller/results.json new file mode 100644 index 0000000000000000000000000000000000000000..60282243e7def09c3258a644f3f87cc87daedb17 --- /dev/null +++ b/data/exp_multi_controller/results.json @@ -0,0 +1,63 @@ +{ + "model": "EleutherAI/pythia-1b", + "n_chunks": 50, + "results": { + "baseline": { + "rope_eps": 0.0, + "attn_eps": 0.0, + "ppl": 15.714571287245102, + "mean_nll": 2.7545883893966674, + "std_nll": 0.2679922705033423, + "elapsed_sec": 6.97636604309082 + }, + "rope_only": { + "rope_eps": 0.5, + "attn_eps": 0.0, + "ppl": 42.76397567251078, + "mean_nll": 3.7556960582733154, + "std_nll": 0.30121930052588414, + "elapsed_sec": 8.34188723564148 + }, + "attn_only": { + "rope_eps": 0.0, + "attn_eps": 0.5, + "ppl": 15.779192556951937, + "mean_nll": 2.758692145347595, + "std_nll": 0.2672898058897529, + "elapsed_sec": 4.9933202266693115 + }, + "combined": { + "rope_eps": 0.5, + "attn_eps": 0.5, + "ppl": 39.49806311144223, + "mean_nll": 3.6762516355514525, + "std_nll": 0.29927777380443615, + "elapsed_sec": 4.9077088832855225 + }, + "rope_strong": { + "rope_eps": 1.0, + "attn_eps": 0.0, + "ppl": 520.7552048869337, + "mean_nll": 6.255280075073242, + "std_nll": 0.373678034428659, + "elapsed_sec": 4.857015609741211 + }, + "attn_strong": { + "rope_eps": 0.0, + "attn_eps": 1.0, + "ppl": 15.912686739617495, + "mean_nll": 2.76711669921875, + "std_nll": 0.2665913433348051, + "elapsed_sec": 4.992188453674316 + }, + "combined_strong": { + "rope_eps": 1.0, + "attn_eps": 1.0, + "ppl": 385.1322818408073, + "mean_nll": 5.953586864471435, + "std_nll": 0.3827877508428578, + "elapsed_sec": 4.99796986579895 + } + }, + "verdict": "WEAK INTERACTION \u2014 combined \u2248 better-of-the-two" +} \ No newline at end of file diff --git a/data/exp_multifractal/EleutherAI--pythia-1.4b_multifractal_v2.json b/data/exp_multifractal/EleutherAI--pythia-1.4b_multifractal_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..95793f5c20be137d4533138877e99d489b35c98f --- /dev/null +++ b/data/exp_multifractal/EleutherAI--pythia-1.4b_multifractal_v2.json @@ -0,0 +1,51 @@ +{ + "model": "EleutherAI/pythia-1.4b", + "n_layers": 24, + "n_heads": 16, + "n_prompts": 50, + "seq_len": 1100, + "summary": { + "PUNCT": { + "median_gamma_good": 0.5863913113085075, + "iqr_gamma_good": 0.7470766537417834, + "n_good": 99, + "n_total": 384, + "mean_r2": 0.27037506225584934 + }, + "COMMON": { + "median_gamma_good": 0.4590440044278438, + "iqr_gamma_good": 1.3502416193894065, + "n_good": 107, + "n_total": 384, + "mean_r2": 0.2678464356612026 + }, + "RARE": { + "median_gamma_good": 0.6703961039342824, + "iqr_gamma_good": 0.6025025010025777, + "n_good": 68, + "n_total": 384, + "mean_r2": 0.21362252484821842 + }, + "NUMERIC": { + "median_gamma_good": 0.8774129186828983, + "iqr_gamma_good": 0.4246948824054274, + "n_good": 201, + "n_total": 384, + "mean_r2": 0.43655141266555847 + }, + "WHITESP": { + "median_gamma_good": 0.8717943197446586, + "iqr_gamma_good": 0.5951920279973467, + "n_good": 137, + "n_total": 384, + "mean_r2": 0.3346697423808716 + }, + "MID": { + "median_gamma_good": 0.4207651222619143, + "iqr_gamma_good": 1.3118979077130546, + "n_good": 113, + "n_total": 384, + "mean_r2": 0.29594510060837814 + } + } +} \ No newline at end of file diff --git a/data/exp_multifractal/EleutherAI--pythia-1b_multifractal.json b/data/exp_multifractal/EleutherAI--pythia-1b_multifractal.json new file mode 100644 index 0000000000000000000000000000000000000000..8b66c1ce8a5e3167666fd35571313968a97de884 --- /dev/null +++ b/data/exp_multifractal/EleutherAI--pythia-1b_multifractal.json @@ -0,0 +1,147 @@ +{ + "model": "EleutherAI/pythia-1b", + "n_prompts": 100, + "seq_len": 1100, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "results_per_class": { + "PUNCT": { + "gamma": 0.2246708151107665, + "r2": 0.2957597952909724, + "n_queries": 67705, + "means_per_d": { + "10": 0.009553949017107329, + "20": 0.004909633214657123, + "30": 0.003418084085778153, + "50": 0.002242935545722749, + "100": 0.0015559094148725923, + "200": 0.0010023041960998578, + "500": 0.0007613012430756139, + "1000": 0.002169432312586328 + } + }, + "COMMON": { + "gamma": 0.12301682332659016, + "r2": 0.08480489378177192, + "n_queries": 176224, + "means_per_d": { + "10": 0.00820340235319055, + "20": 0.004018882758558712, + "30": 0.0027472936608260467, + "50": 0.0019313373152108613, + "100": 0.0015103531072477479, + "200": 0.000907947002256553, + "500": 0.0007121923696445134, + "1000": 0.0028085027561016752 + } + }, + "RARE": { + "gamma": -0.11172827606855157, + "r2": 0.038377803607796857, + "n_queries": 19681, + "means_per_d": { + "10": 0.006779983114670504, + "20": 0.0034277609300277373, + "30": 0.0024965771438588227, + "50": 0.001818577710485751, + "100": 0.001592096605244473, + "200": 0.0009472472142429144, + "500": 0.0008539385590816568, + "1000": 0.0070645468378508535 + } + }, + "NUMERIC": { + "gamma": 0.15279454850018664, + "r2": 0.12606423897305286, + "n_queries": 11456, + "means_per_d": { + "10": 0.008487805864280294, + "20": 0.0036661966841048956, + "30": 0.002659999558170761, + "50": 0.001936049460430504, + "100": 0.0013715839043992465, + "200": 0.0009419402108453574, + "500": 0.0006142259852306263, + "1000": 0.002518095334984621 + } + }, + "WHITESP": { + "gamma": 0.5363952831125038, + "r2": 0.7613349332626967, + "n_queries": 3108, + "means_per_d": { + "10": 0.003930726958617291, + "20": 0.0036198270849574447, + "30": 0.0024071913079301344, + "50": 0.002066257173922952, + "100": 0.0022580054510070616, + "200": 0.0015699543872801196, + "500": 0.0010837609974705443, + "1000": 0.00027048688823893917 + } + }, + "MID": { + "gamma": 0.10735627775655544, + "r2": 0.09026029816283054, + "n_queries": 189826, + "means_per_d": { + "10": 0.007354267455411016, + "20": 0.003749796488878495, + "30": 0.00256299730998833, + "50": 0.0019151222754204699, + "100": 0.0015491446045259422, + "200": 0.0009095697220157391, + "500": 0.0008648605044025502, + "1000": 0.0025259429366056817 + } + } + }, + "summary": { + "PUNCT": { + "gamma": 0.2246708151107665, + "r2": 0.2957597952909724, + "n": 67705 + }, + "COMMON": { + "gamma": 0.12301682332659016, + "r2": 0.08480489378177192, + "n": 176224 + }, + "RARE": { + "gamma": -0.11172827606855157, + "r2": 0.038377803607796857, + "n": 19681 + }, + "NUMERIC": { + "gamma": 0.15279454850018664, + "r2": 0.12606423897305286, + "n": 11456 + }, + "WHITESP": { + "gamma": 0.5363952831125038, + "r2": 0.7613349332626967, + "n": 3108 + }, + "MID": { + "gamma": 0.10735627775655544, + "r2": 0.09026029816283054, + "n": 189826 + } + } +} \ No newline at end of file diff --git a/data/exp_multifractal/EleutherAI--pythia-1b_multifractal_v2.json b/data/exp_multifractal/EleutherAI--pythia-1b_multifractal_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..a313ab5faa5b10ca3528e71fe1d93ebfd71df561 --- /dev/null +++ b/data/exp_multifractal/EleutherAI--pythia-1b_multifractal_v2.json @@ -0,0 +1,3015 @@ +{ + "model": "EleutherAI/pythia-1b", + "n_prompts": 50, + "seq_len": 1100, + "distances": [ + 10, + 20, + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "fit_distances": [ + 30, + 50, + 100, + 200, + 500, + 1000 + ], + "per_class_summary": { + "PUNCT": { + "n_heads_total": 128, + "n_heads_finite": 128, + "n_heads_good_r2": 50, + "median_gamma_all": 0.21079186463698427, + "median_gamma_good_r2": 0.6129411870843091, + "iqr_gamma_good_r2": 0.6834391935356313, + "mean_r2": 0.4121555877547386, + "total_n_queries": 4193981 + }, + "COMMON": { + "n_heads_total": 128, + "n_heads_finite": 128, + "n_heads_good_r2": 39, + "median_gamma_all": 0.09574768130399607, + "median_gamma_good_r2": 0.7549939604641029, + "iqr_gamma_good_r2": 1.4474947259735251, + "mean_r2": 0.3513391182674777, + "total_n_queries": 11289806 + }, + "RARE": { + "n_heads_total": 128, + "n_heads_finite": 127, + "n_heads_good_r2": 32, + "median_gamma_all": -0.08733986175560474, + "median_gamma_good_r2": 0.8537601723990879, + "iqr_gamma_good_r2": 0.5654723557231875, + "mean_r2": 0.3100526277698701, + "total_n_queries": 1235341 + }, + "NUMERIC": { + "n_heads_total": 128, + "n_heads_finite": 127, + "n_heads_good_r2": 113, + "median_gamma_all": 0.8918778471829276, + "median_gamma_good_r2": 0.92140228475535, + "iqr_gamma_good_r2": 0.4913766592022941, + "mean_r2": 0.7387979846706858, + "total_n_queries": 680348 + }, + "WHITESP": { + "n_heads_total": 128, + "n_heads_finite": 127, + "n_heads_good_r2": 92, + "median_gamma_all": 0.7215592120737407, + "median_gamma_good_r2": 0.8303539334063064, + "iqr_gamma_good_r2": 0.5451031266992266, + "mean_r2": 0.6093644712590012, + "total_n_queries": 202900 + }, + "MID": { + "n_heads_total": 128, + "n_heads_finite": 128, + "n_heads_good_r2": 47, + "median_gamma_all": 0.021432221351514957, + "median_gamma_good_r2": 0.6511505208150729, + "iqr_gamma_good_r2": 1.3401231938732152, + "mean_r2": 0.3849162555065023, + "total_n_queries": 11964094 + } + }, + "per_class_per_head": { + "PUNCT": { + "gamma": [ + [ + 0.9550858442760598, + 1.1865401196351664, + 0.7873046695566263, + 1.2669164653733398, + 2.0370173043338666, + 1.324262950029752, + 0.7920197617548389, + 1.2916652303269 + ], + [ + 1.694396553408639, + 2.025224043134131, + 0.6185159921192004, + 2.1528040259602013, + 0.7380153065828226, + 0.6075833580148465, + 0.8619702364391326, + 0.8778144272035998 + ], + [ + 1.2564629717241762, + 0.9819937433339981, + 1.1679269770446068, + 0.5549358324693364, + 0.5606082006356339, + 0.9670643455860445, + 0.5498188303093874, + 1.0997415931280392 + ], + [ + 0.5072847715025453, + 1.0000265018691037, + 1.389318730205955, + 0.6463568823171995, + 1.00332949570489, + -0.5079133039023745, + 0.6182990161537716, + 0.39428568910089024 + ], + [ + 0.11591410564443855, + -0.2345938665271384, + 0.2753593533299013, + -0.020176880309775363, + 0.08824237046475493, + 0.05591441670029464, + 0.30294434228789624, + -0.026800706267301456 + ], + [ + 0.38764321895252385, + 0.39886677339546783, + 0.3696729459492863, + -0.05094607305288471, + 0.3105357658800853, + -0.09911294894955612, + 0.23026824145460956, + 0.08033153575819742 + ], + [ + -0.5476757290509705, + 0.2650485411364192, + 0.22661791759008415, + 0.1579924501240579, + 0.3157547601813388, + 0.15797022777588385, + 0.15412951104624897, + -0.03036876374277142 + ], + [ + 0.2094515734779761, + -0.36116301601523676, + 0.13238338865841492, + -0.05566570361091307, + 1.4779891555086826, + -0.3325129977679278, + -0.19689968313801945, + 0.2475853738265866 + ], + [ + 0.022833942213868915, + 0.3899769091836714, + 0.4552397942330174, + 0.3013908402913576, + 0.1298754684101375, + 0.14950380770911278, + 0.4130359495748237, + 0.1302564769846052 + ], + [ + 0.3420201313533788, + 0.26753984577182316, + -0.27825045608069837, + 0.4289194831052632, + 0.4074825238525051, + 0.26294846188995846, + 0.32031273785979725, + 0.28227318646568816 + ], + [ + 0.1324335554741208, + -0.24659844552030985, + 0.05282929253033652, + 0.24495365903573155, + 0.23315142772738884, + 0.13374952027807724, + -0.10524819059341724, + -0.05418296525420075 + ], + [ + 0.16427440804989477, + -0.08974145613314588, + 0.0415824933304311, + 0.09821541198122202, + 0.0909329044621399, + 0.0270318711518888, + 0.03483765210309106, + 0.08538920850199591 + ], + [ + 0.16740499406007006, + 0.1080103232821023, + -0.0754162272168228, + 0.11430577956602482, + -0.015168935044080843, + -0.2789792155714718, + 0.04011689683458933, + 0.1590802299710024 + ], + [ + -0.34104645156469765, + 0.2449393264507771, + 0.09214774191164353, + 0.16556273917209582, + -0.043301627757932136, + -0.09108758850917018, + 0.10354096652529453, + -0.038926034642272324 + ], + [ + -0.44213754568738245, + 0.19525778446048442, + 0.4220991196457288, + 0.2026701836699308, + 0.2456771609451869, + -0.0352148582562926, + 0.21722022206575334, + 0.09450765894794598 + ], + [ + -0.01949724657577062, + 0.44915647413104776, + 0.011927974183662574, + 0.28012574846060245, + 0.1333597928862481, + -0.06877897477399951, + 0.21213215579599243, + 0.30289136532952615 + ] + ], + "r2": [ + [ + 0.9729886759976621, + 0.9902498705177085, + 0.8837207943607049, + 0.9512377524089113, + 0.9646583760034071, + 0.937948023964508, + 0.9732417891815094, + 0.9901626651081823 + ], + [ + 0.5795370857438771, + 0.9856585509697481, + 0.963291891328334, + 0.7934327595402201, + 0.3359220269610125, + 0.9785507071368438, + 0.8546365801966401, + 0.9283080357549374 + ], + [ + 0.8741136877311652, + 0.6089703018943748, + 0.8541495826660366, + 0.5009262313647718, + 0.9765615664722426, + 0.6194620089201365, + 0.6844204675759087, + 0.9932673740341769 + ], + [ + 0.9595810308109548, + 0.9808060566922151, + 0.9425001722809665, + 0.7408967471307908, + 0.7790752414332069, + 0.24766973927053937, + 0.5212611822977395, + 0.287129783224755 + ], + [ + 0.08803066503863488, + 0.1475267338178966, + 0.4311709293090813, + 0.0033736854184689413, + 0.10744498462547636, + 0.008108964539091601, + 0.2902448392992071, + 0.0024925045917595368 + ], + [ + 0.29187700025115715, + 0.7930258348801245, + 0.7930811347536488, + 0.0062256371750740724, + 0.6380944968006863, + 0.039173148136070646, + 0.6648604957950115, + 0.0415532885907729 + ], + [ + 0.601982380052535, + 0.5255144491047874, + 0.34570669983685953, + 0.33375199437391057, + 0.33490617862579375, + 0.07297401216206134, + 0.16541316894581515, + 0.008437122542073516 + ], + [ + 0.7434381534552339, + 0.6781384502530483, + 0.39318928759481386, + 0.012337943728491951, + 0.9801762613180655, + 0.366668640167104, + 0.12875336065756227, + 0.22476401893169817 + ], + [ + 0.017511074664334214, + 0.7710883213294528, + 0.7146267179589392, + 0.21195309753171332, + 0.10799262844419144, + 0.11631330812782736, + 0.7517782304484504, + 0.06885792916673983 + ], + [ + 0.9332467366018941, + 0.5456956989456243, + 0.2580796141375211, + 0.5178314348491422, + 0.839139721871739, + 0.8763209717952417, + 0.2164367846892209, + 0.29084067037489114 + ], + [ + 0.16920787517359515, + 0.22662540051680402, + 0.02830254461267434, + 0.30088309471659447, + 0.1880542210659254, + 0.3290501799882286, + 0.19970355654347804, + 0.017513701266740345 + ], + [ + 0.295525811319871, + 0.05973844334603129, + 0.016359181265723666, + 0.13417055785254972, + 0.03691534504551097, + 0.012333571005980648, + 0.02825101668610608, + 0.08732398621818405 + ], + [ + 0.3076748311860761, + 0.06337853282042882, + 0.0322290268864075, + 0.1559588377273179, + 0.0009944374084700502, + 0.37821676148118133, + 0.03920828828315093, + 0.4478577305006617 + ], + [ + 0.34971462611636606, + 0.38131734075990975, + 0.09909997990425456, + 0.19856554877606392, + 0.012743753323226525, + 0.07879548745725418, + 0.3259868663755414, + 0.007082418931514622 + ], + [ + 0.45640419553872724, + 0.2026259653448892, + 0.6341304427657783, + 0.13291124053853165, + 0.31239733004555625, + 0.005633925474795132, + 0.37104077439075855, + 0.03499074967509741 + ], + [ + 0.002484427566799785, + 0.6448736010610299, + 0.0007009950835804579, + 0.7178230916101758, + 0.15847690262196978, + 0.028737745759191635, + 0.5317454850241488, + 0.3576692408605654 + ] + ], + "n": [ + [ + 33252, + 33252, + 33252, + 33251, + 33250, + 33252, + 33252, + 33251 + ], + [ + 31132, + 33252, + 33249, + 33204, + 30098, + 33252, + 33252, + 33252 + ], + [ + 33233, + 33209, + 33225, + 33252, + 33252, + 33252, + 33252, + 33252 + ], + [ + 33252, + 33252, + 33137, + 33252, + 33244, + 23, + 33059, + 33209 + ], + [ + 33252, + 33111, + 33252, + 33216, + 32936, + 33172, + 33252, + 32819 + ], + [ + 32997, + 33168, + 33252, + 32718, + 33252, + 33166, + 33252, + 33209 + ], + [ + 32428, + 33252, + 33252, + 22704, + 33249, + 32928, + 33247, + 33252 + ], + [ + 33244, + 32928, + 33243, + 32758, + 33220, + 32575, + 32521, + 33247 + ], + [ + 33252, + 33252, + 33248, + 32925, + 33235, + 33191, + 33251, + 33195 + ], + [ + 33252, + 33252, + 32400, + 33207, + 33252, + 33251, + 32862, + 33243 + ], + [ + 33243, + 32458, + 33252, + 33252, + 33125, + 33247, + 33231, + 33252 + ], + [ + 33252, + 33245, + 33224, + 33252, + 33228, + 33249, + 33243, + 33243 + ], + [ + 33246, + 33251, + 33171, + 33248, + 33245, + 33252, + 33252, + 33252 + ], + [ + 33062, + 33241, + 33252, + 33242, + 31893, + 33252, + 33252, + 33166 + ], + [ + 32931, + 33042, + 33152, + 32954, + 33227, + 32425, + 33252, + 33157 + ], + [ + 32960, + 33224, + 33209, + 33243, + 33250, + 33242, + 33250, + 32792 + ] + ] + }, + "COMMON": { + "gamma": [ + [ + 0.9716522396471736, + 1.1308964592164885, + 0.8358113650289176, + 1.2066153172111167, + 2.0158573486801843, + 1.2460466264579724, + 0.8058158528046667, + 1.298332573909185 + ], + [ + 1.2866394254076658, + 1.8752432296994312, + 0.6610362098954083, + 2.1157558043094977, + 0.684702189482019, + 0.5927952964835315, + 0.8525869866078309, + 0.7549939604641029 + ], + [ + 1.1696215148696485, + 1.0543923056253663, + 0.867861761757508, + 0.5111834429979047, + 0.6902386610528892, + 0.8300337169661837, + 0.4544547486541997, + 0.9463898209818001 + ], + [ + 0.5430230651744012, + 1.0836642779325318, + 1.1233812933790257, + 0.5981370280039285, + 0.6388536056349813, + 0.9338915015825332, + 0.9095714119734413, + 0.4596086789493678 + ], + [ + 0.25426889907529976, + -0.24192292750647465, + 0.03530292489726423, + -0.10685758785784381, + -0.15771234707114407, + -0.2693600370449772, + 0.08712065381111711, + -0.16199261821455654 + ], + [ + 0.18560202977972573, + 0.04317997402090628, + 0.1408535165066153, + -0.2593475774848983, + 0.11021033289027066, + -0.5985621946907574, + -0.0752009492511234, + 0.07266091724690497 + ], + [ + -0.533976857940842, + 0.08008605680867742, + 0.15769697434619803, + 0.2546377626802753, + 0.22650876595046843, + -0.017631781605909228, + 0.2618937634597779, + 0.008504445644071223 + ], + [ + -0.11387166838276451, + -0.32140035423270175, + -0.2610822577379707, + -0.4746216128107331, + 1.4797428607661602, + -0.43865722258638573, + -0.44298716316324405, + 0.1791030681805478 + ], + [ + -0.11400041333190412, + 0.2324853955098993, + 0.2947510243635586, + 0.23948358432835476, + 0.15685289025680504, + 0.11915931905161886, + 0.2462552787612185, + 0.07770191307752626 + ], + [ + 0.16930415857406947, + 0.07093596119943509, + -0.13677065305520736, + 0.34699622055250434, + 0.21900380480895995, + 0.14299736381284173, + 0.23505731941251604, + 0.3952982936713913 + ], + [ + -0.13343952192538297, + -0.2507220118144093, + 0.0005592565846895209, + 0.14951450945342268, + 0.19357581910316518, + -0.03803086401519909, + -0.22799465162198163, + 0.08351552332186725 + ], + [ + 0.17752684104479527, + 0.08731804002771015, + 0.014057107800459, + 0.06184507233601859, + 0.2419056439062526, + -0.02173333108668324, + -0.044065254179437935, + 0.10143536157853439 + ], + [ + 0.036825169349414155, + 0.17875276670719456, + -0.085355255042993, + 0.020097616720742645, + -0.052553626796928675, + -0.19278992393863606, + -0.028135020806488373, + 0.033196535647210514 + ], + [ + -0.21208797673542715, + 0.17143803969855242, + -0.03309977540970705, + 0.04477577345932552, + -0.39120504136397577, + -0.11884290821823612, + -0.02644624415271322, + -0.44312928177554134 + ], + [ + -0.32795928274952685, + -0.10575649895979235, + 0.12744925058681905, + -0.2518590238673466, + 0.02761491318219541, + -0.41414909953099277, + 0.1078315187380176, + 0.00025954490635389716 + ], + [ + -0.3911058457462157, + 0.06831609989993023, + -0.3665435264027911, + 0.06127961970623372, + 0.13914277766400845, + -0.35932775146396717, + 0.09006000102945774, + -0.1783154471446204 + ] + ], + "r2": [ + [ + 0.9776916364769619, + 0.9784339520270704, + 0.8784867627625983, + 0.9433906112875925, + 0.9896875813528969, + 0.9584461261367543, + 0.9823432355750267, + 0.9843088223733454 + ], + [ + 0.384135614888909, + 0.9817430369395836, + 0.9243770262693835, + 0.7879534356861859, + 0.3282671496543954, + 0.9530570074783606, + 0.853994806458166, + 0.9743603525599038 + ], + [ + 0.9641535043256172, + 0.4961573042033258, + 0.8042773463855257, + 0.36801365981204925, + 0.9800250250914634, + 0.6050273489411452, + 0.5284393751144658, + 0.9930002503008635 + ], + [ + 0.9651892925244434, + 0.994900442366395, + 0.8854021733078271, + 0.7212955321653789, + 0.7131808995341, + 0.26597297017768007, + 0.7730667161659295, + 0.4496512753273355 + ], + [ + 0.27398050980652666, + 0.1913475720390999, + 0.00997392971465716, + 0.12014186173039187, + 0.26392622742751937, + 0.26702786393017763, + 0.030428217332320817, + 0.09844833493011607 + ], + [ + 0.09106886450289264, + 0.01961672996700292, + 0.19549606452663149, + 0.24684041857521022, + 0.08888487525341171, + 0.7771523892397014, + 0.07400493554832821, + 0.023663048582568957 + ], + [ + 0.7867357411541903, + 0.03407963886456533, + 0.12259476837129324, + 0.2048422071601853, + 0.1696966005515117, + 0.0010103338016713703, + 0.28682677304429516, + 0.0004520681372214286 + ], + [ + 0.12513233313932615, + 0.6137663055836697, + 0.4426623063539097, + 0.6842450589838445, + 0.9472768345097538, + 0.5701418376446898, + 0.5917920755521004, + 0.12492027650513904 + ], + [ + 0.1569333164775698, + 0.3104976592645081, + 0.3741178374776223, + 0.14647997100178622, + 0.10556706784277348, + 0.07212282315020035, + 0.3017586077725706, + 0.021156003010135316 + ], + [ + 0.33877971914400573, + 0.02875016260849106, + 0.10029502615715524, + 0.2872323603171306, + 0.3364090530564443, + 0.4929447742546542, + 0.1572314052628614, + 0.5160175516488207 + ], + [ + 0.12242520329023043, + 0.29423937573357317, + 2.3554302346351363e-06, + 0.10453024853518622, + 0.129570314950316, + 0.017570950844239208, + 0.4095601077988005, + 0.03747297360460611 + ], + [ + 0.22486584029280432, + 0.06193756078992296, + 0.0015956672880589329, + 0.026135546824363698, + 0.21674076136934672, + 0.004767591098300161, + 0.021456408075243627, + 0.08401509380438987 + ], + [ + 0.011320507835489435, + 0.14982266453696724, + 0.03830672782546107, + 0.0037599882352795655, + 0.011829117103060671, + 0.22573684753725975, + 0.00876128513895369, + 0.020896639111448456 + ], + [ + 0.1702016833380796, + 0.18554451757344537, + 0.008480852460092225, + 0.013353815466593533, + 0.5423088191172282, + 0.08685935255130739, + 0.01049713594479973, + 0.6551968921028201 + ], + [ + 0.32318319798061734, + 0.051580586906335935, + 0.07857609180045022, + 0.26304141151837, + 0.0035772110791920664, + 0.7281047315654099, + 0.1049370265694014, + 2.5805670744727394e-07 + ], + [ + 0.506215112932759, + 0.016574846540944765, + 0.5576173695475293, + 0.030500223752128797, + 0.12337099760591641, + 0.4673198369116216, + 0.07879903891398232, + 0.12534573636654323 + ] + ], + "n": [ + [ + 89343, + 89320, + 89343, + 89320, + 89343, + 89343, + 89343, + 89334 + ], + [ + 78549, + 89343, + 89228, + 88968, + 77084, + 89343, + 89343, + 89343 + ], + [ + 89223, + 89133, + 89342, + 89342, + 89343, + 89343, + 89343, + 89343 + ], + [ + 89343, + 89343, + 89322, + 89315, + 89340, + 123, + 88250, + 89266 + ], + [ + 89342, + 89020, + 89343, + 88419, + 88401, + 89130, + 89341, + 88868 + ], + [ + 89168, + 89260, + 89343, + 88780, + 89343, + 88968, + 89343, + 88914 + ], + [ + 87982, + 89333, + 89343, + 79672, + 89334, + 89224, + 89320, + 89301 + ], + [ + 89317, + 88292, + 89333, + 88185, + 89287, + 87746, + 88524, + 89340 + ], + [ + 89343, + 89343, + 89341, + 89040, + 89305, + 89178, + 89336, + 89284 + ], + [ + 89343, + 89333, + 87962, + 89283, + 89343, + 89342, + 88730, + 89285 + ], + [ + 89339, + 89111, + 89343, + 89340, + 89173, + 89323, + 89245, + 89343 + ], + [ + 89340, + 89304, + 89292, + 89343, + 89210, + 89328, + 89320, + 89290 + ], + [ + 89338, + 89343, + 89035, + 89331, + 89323, + 89343, + 89340, + 89342 + ], + [ + 89120, + 89342, + 89335, + 89318, + 88491, + 89343, + 89341, + 89198 + ], + [ + 88726, + 89017, + 89214, + 89025, + 89135, + 86253, + 89340, + 89206 + ], + [ + 88963, + 89216, + 89234, + 89327, + 89342, + 89243, + 89340, + 88787 + ] + ] + }, + "RARE": { + "gamma": [ + [ + 1.0551550411543253, + 1.1507132642623739, + 0.8690510511612634, + 1.4323517207467247, + 1.7958014863998164, + 1.2250249050723103, + 0.8019597802801429, + 1.1332952083668355 + ], + [ + 1.3001203230581215, + 2.0255883217114117, + 0.4878622424664753, + 2.36044575623729, + 0.7993890142187439, + 0.46890704832446095, + 0.9489372533336315, + 0.8170113584678095 + ], + [ + 0.8384692936369124, + 1.105619028640761, + 0.7580601745687914, + 0.608328593970791, + 0.6762981317625788, + 1.0908026485116467, + 0.7157473702023209, + 0.8183954928628856 + ], + [ + 0.5111972281284427, + 1.0260001358185344, + 1.1686949919435723, + 0.47124417584715605, + 0.5925040794472292, + NaN, + 0.9302989782678229, + 0.5357496624460014 + ], + [ + 0.10501379054030897, + -0.17750089732715452, + 0.062079172638176985, + -0.35262352726059126, + -0.32910224033334967, + -0.26787174984685547, + 0.05419424223920609, + -0.13370015087781317 + ], + [ + -0.0007756853588246551, + -0.11831654202693573, + -0.007524048991582557, + -0.5330857499554105, + 0.032755280839689566, + -0.9502652237079496, + -0.2401808598620147, + -0.02683958492993383 + ], + [ + -0.5805212598604513, + 0.06738280422752865, + 0.16348381480710653, + 0.13531870474565935, + 0.13692954745352448, + -0.08919711238907833, + 0.10504345544454323, + -0.18052628029556597 + ], + [ + -0.25521723872997476, + -0.4754565349445725, + -0.21462705317873304, + -0.7023839198812671, + 1.5092118773355097, + -0.6597498045159653, + -0.796433452149296, + 0.11498075997890289 + ], + [ + -0.16419301899534605, + 0.027363181922400413, + 0.0824667484389805, + -0.09262690354715945, + 0.007931071714500488, + -0.12729819619829724, + 0.0017738757022179303, + -0.15480175798896484 + ], + [ + 0.016100278845672052, + -0.0741894516475177, + -0.17475303830369407, + 0.05835929601339684, + -0.032684598584426294, + 0.017471815200417382, + -0.08160367406352839, + 0.06472064346708925 + ], + [ + -0.2011746755909363, + -0.4743183564215081, + -0.08341815466502052, + -0.10738255325086293, + -0.09152861271518659, + -0.18904277747347972, + -0.3242694419899233, + 0.09703629130692863 + ], + [ + 0.003046903772439052, + -0.0374683910734031, + -0.2086961152927192, + -0.11292645608558427, + -0.08196983568870866, + -0.23841626565189475, + -0.20522432495299056, + -0.15365220924194714 + ], + [ + -0.22607050554374522, + -0.09323389723256384, + -0.3072160003714468, + -0.16206284835270174, + -0.330510949714584, + -0.2550399466317629, + -0.1230570629156563, + 0.005230170677590375 + ], + [ + -0.44238198169017323, + -0.08733986175560474, + -0.27613491930216, + -0.18763175337718177, + -0.6154609577082006, + -0.24769761779643565, + -0.10508998584646913, + -0.5413179282362163 + ], + [ + -0.5916250403869138, + -0.2775703523903287, + -0.09819170804428667, + -0.5482230905155601, + -0.1629400968399668, + -0.4459334311673832, + -0.12132318443606611, + -0.16474402825384954 + ], + [ + -0.5018874901692985, + -0.22165894459570143, + -0.2440435565794564, + -0.11513155398519442, + -0.029391676961782593, + -0.22529884387330038, + -0.1134402390870844, + -0.2685325271683713 + ] + ], + "r2": [ + [ + 0.9682504678790873, + 0.9715934551952228, + 0.8896279270626272, + 0.9277688225115417, + 0.9739386096641234, + 0.9727271809524155, + 0.9825568300275945, + 0.9455231407495659 + ], + [ + 0.48068149469003085, + 0.9828371022433677, + 0.9068881594830885, + 0.8358646994814145, + 0.4300476152661672, + 0.807787029261795, + 0.782302907332641, + 0.9474246713553416 + ], + [ + 0.932986526413252, + 0.5362832614906935, + 0.7740136949555425, + 0.4993985742408771, + 0.9545973866763369, + 0.7429027501741694, + 0.8420856249385069, + 0.9807513950348495 + ], + [ + 0.9590784433895165, + 0.9858678799625396, + 0.9308532601010366, + 0.7143112056895023, + 0.69411940498279, + 0.0, + 0.6621897158342269, + 0.4957661685857363 + ], + [ + 0.05769889808373596, + 0.049362875227030645, + 0.015527223584729466, + 0.41077598915620783, + 0.37703939123449925, + 0.13834748548898634, + 0.009069319020753697, + 0.03143539227219472 + ], + [ + 8.894538801040852e-07, + 0.13700985341868288, + 0.0002464640206064628, + 0.41719808345807796, + 0.005370424873748836, + 0.6863149873437585, + 0.2329270816438097, + 0.0022722962983505735 + ], + [ + 0.293410514283733, + 0.01439549330059875, + 0.08392726771265702, + 0.026350790456847295, + 0.05024308547995815, + 0.013306535423866506, + 0.056610894282959956, + 0.1709667094240518 + ], + [ + 0.2926889861408488, + 0.5674455699309039, + 0.14165168967883845, + 0.5663334928913719, + 0.9379092955788946, + 0.4335098353917133, + 0.5485889445096309, + 0.041566909172514044 + ], + [ + 0.14155623492567415, + 0.0034179846769669675, + 0.025403500824885072, + 0.01516469206089155, + 0.00014332155056184792, + 0.06329058789768638, + 1.4560927487639042e-05, + 0.05692110410318951 + ], + [ + 0.0019418592811037971, + 0.017520868944902235, + 0.043097874677532566, + 0.007228302087620597, + 0.005630355213568583, + 0.0027912905696444446, + 0.01254351586653446, + 0.011947437738008881 + ], + [ + 0.13318946281645005, + 0.3562266272233786, + 0.03665509654658705, + 0.03815600003758224, + 0.019814959877594185, + 0.14324041684051236, + 0.30830195720566855, + 0.0246728065009012 + ], + [ + 3.4580970159958646e-05, + 0.004089637246200151, + 0.1149504457268904, + 0.04338589799561998, + 0.012787491502453285, + 0.1737223048172818, + 0.17354685535654457, + 0.09116015227209828 + ], + [ + 0.1558104629956376, + 0.021876661608009496, + 0.25604284214469797, + 0.09361436731818196, + 0.2223404472064695, + 0.13522497440201375, + 0.09008600517695842, + 0.000195444388970345 + ], + [ + 0.28760501481610556, + 0.03406846920930129, + 0.22577911958032149, + 0.11395893762758369, + 0.4720682913402119, + 0.14999763244333275, + 0.058282049717198525, + 0.4027255328803423 + ], + [ + 0.4453869750972872, + 0.1509746335534976, + 0.027088049379767498, + 0.46785174375415295, + 0.0547745532123759, + 0.4719401616108495, + 0.0721566797711537, + 0.04974526052625183 + ], + [ + 0.357502861782958, + 0.0737146797571866, + 0.07135215313850563, + 0.051467313794970604, + 0.002127760460607253, + 0.10201435546351145, + 0.050049184848352524, + 0.11583747739038608 + ] + ], + "n": [ + [ + 9773, + 9773, + 9773, + 9773, + 9773, + 9773, + 9773, + 9773 + ], + [ + 9476, + 9773, + 9773, + 9721, + 9137, + 9773, + 9773, + 9773 + ], + [ + 9769, + 9764, + 9773, + 9773, + 9773, + 9772, + 9773, + 9773 + ], + [ + 9773, + 9773, + 9772, + 9773, + 9773, + 2, + 9700, + 9766 + ], + [ + 9773, + 9698, + 9773, + 9433, + 9589, + 9748, + 9773, + 9712 + ], + [ + 9773, + 9764, + 9773, + 9751, + 9773, + 9724, + 9773, + 9703 + ], + [ + 9506, + 9771, + 9773, + 8666, + 9772, + 9763, + 9764, + 9766 + ], + [ + 9763, + 9508, + 9772, + 9616, + 9755, + 9532, + 9622, + 9772 + ], + [ + 9773, + 9773, + 9773, + 9740, + 9771, + 9770, + 9772, + 9771 + ], + [ + 9773, + 9773, + 9606, + 9765, + 9773, + 9773, + 9729, + 9768 + ], + [ + 9771, + 9746, + 9773, + 9773, + 9761, + 9767, + 9761, + 9773 + ], + [ + 9773, + 9771, + 9772, + 9773, + 9766, + 9772, + 9773, + 9768 + ], + [ + 9773, + 9770, + 9737, + 9773, + 9768, + 9773, + 9770, + 9773 + ], + [ + 9732, + 9773, + 9772, + 9765, + 9602, + 9770, + 9773, + 9713 + ], + [ + 9617, + 9710, + 9727, + 9703, + 9741, + 9389, + 9773, + 9759 + ], + [ + 9696, + 9757, + 9758, + 9765, + 9773, + 9763, + 9773, + 9665 + ] + ] + }, + "NUMERIC": { + "gamma": [ + [ + 1.036648901533842, + 1.097278462485711, + 0.9572901731112674, + 1.2762356341605, + 2.168162873425152, + 1.236965820322021, + 0.7710861510491587, + 1.2407123611948916 + ], + [ + 1.0195176694689656, + 2.1075753846037113, + 0.44624404402649237, + 2.7309572242734976, + 0.7566594024276964, + 0.543712261409624, + 0.862841056094304, + 0.8467032818459578 + ], + [ + 1.1222198453613381, + 1.0974066068706545, + 0.9073568371081262, + 0.6522543483867755, + 0.629050066761155, + 1.203680679088676, + 0.8663922207947405, + 0.8668190376163205 + ], + [ + 0.43774127561331416, + 1.009801915310458, + 1.4336725150109257, + 0.6364026730433038, + 0.7696706404568014, + NaN, + 1.01226099538308, + 0.5939324209712102 + ], + [ + 0.7546148168538105, + 1.415295978473062, + 0.7838654204041644, + 0.6413615744934928, + -0.14772169724690765, + 0.9873024938937157, + 0.9542526871906855, + 1.6102208088498529 + ], + [ + 1.652510003586582, + 1.049034677070183, + 0.6207728879924047, + 1.2596577039921593, + 0.7779728849749975, + 0.4294703879405925, + 0.51504367819282, + 0.6575837192571141 + ], + [ + 1.158608837869695, + 0.9964577034069263, + 1.0252049388047968, + -0.012303704036747886, + 1.1556322789644808, + 1.4344488145412995, + 0.8918778471829276, + 0.9588260904935508 + ], + [ + 0.6291237177545528, + 0.2771175436963159, + 0.8568494664142416, + 0.9096976117265335, + 1.9068555983173254, + 1.2956384924447666, + 1.688652162454101, + 1.0488598316786821 + ], + [ + 0.6846293973839902, + 0.7710787841147115, + 0.92140228475535, + 0.9628679699213053, + 1.0634993230037975, + 0.9320838196513533, + 0.9668381746953877, + 1.1755319769335237 + ], + [ + 0.508942846986228, + 0.650156021963115, + 1.4291136447381294, + 1.089638205562059, + 0.8228914677114125, + 0.5087889856965601, + 1.3416465225497345, + 0.8070782467771738 + ], + [ + 0.5671916628678744, + 1.328272558778496, + 0.7173427906192331, + 1.0368783599732954, + 1.4075014097852556, + 0.8083209739781254, + 0.6367570310502737, + 0.9358901646026638 + ], + [ + 0.6992444270065075, + 0.7178126784754151, + 0.7561464137052035, + 0.725463678973236, + 0.9349739146584473, + 0.6098657424871817, + 0.6641100489284115, + 0.8771827241338246 + ], + [ + 0.6276653508760355, + 0.8845532772075229, + 0.7543242609240222, + 0.5785181228241558, + 1.1518941153226174, + 0.6625318133149156, + 0.32245106801456697, + 0.37861148984828513 + ], + [ + 2.0026620040311354, + 0.6097129271114715, + 0.5058677086286061, + 0.5631080347350422, + 1.0502678164757588, + 0.6835279391368037, + 0.3249773643745697, + 0.8115601157612699 + ], + [ + 1.7062158872409228, + 1.3119934521438399, + 1.2355302782670754, + 1.5730377429834161, + 1.027773179533541, + 0.8584903712148152, + 0.5891141628018248, + 1.4239961208158527 + ], + [ + 1.0248527521732496, + 1.1123872502910117, + 1.1539084725172097, + 0.5992528209801854, + 0.5683931395715358, + 1.0049707419461131, + 0.6095139063447402, + 1.1775114837129508 + ] + ], + "r2": [ + [ + 0.9781377419597952, + 0.9948361773508599, + 0.8497817298022252, + 0.8720417214333321, + 0.9761439865076095, + 0.9598552421137175, + 0.9772455077599782, + 0.9750584684826007 + ], + [ + 0.20853982307457652, + 0.9850433931296063, + 0.7702829222855512, + 0.8465154327710747, + 0.4673920161866403, + 0.9023355987565668, + 0.8040228339427722, + 0.9471599804355888 + ], + [ + 0.9356342664279809, + 0.502162079517364, + 0.9095628289646726, + 0.6180927497710567, + 0.9178852429031825, + 0.8213926756645111, + 0.8380546098771053, + 0.928019056406843 + ], + [ + 0.9010196658389661, + 0.9315942928729629, + 0.9415483738654578, + 0.7663083543752847, + 0.732482647338913, + 0.0, + 0.5575403969814285, + 0.3836536689030893 + ], + [ + 0.8055182785101879, + 0.8271045627561351, + 0.7688311031611892, + 0.41358580506980425, + 0.18341186061983128, + 0.7509569406468375, + 0.9172694944791311, + 0.699280102082619 + ], + [ + 0.9324155704681504, + 0.8211989862077484, + 0.8480467564059665, + 0.7127416413099132, + 0.881635353403216, + 0.07015176755741448, + 0.8014713685639487, + 0.6581603235691248 + ], + [ + 0.3614593152232908, + 0.9790481438725837, + 0.8767806386219353, + 7.110195384507634e-05, + 0.863772876798291, + 0.7351659527373972, + 0.8536710633692854, + 0.7318099701932845 + ], + [ + 0.7959494008168851, + 0.11557895009533037, + 0.8602746137988291, + 0.3088546589306099, + 0.9842710160094892, + 0.3776047542936072, + 0.6320565517735478, + 0.8261841477796318 + ], + [ + 0.8040515508058105, + 0.8870810486176645, + 0.9335504139388461, + 0.7238420398419104, + 0.7723423645231097, + 0.8724990440530581, + 0.8834825304236281, + 0.8594323105335662 + ], + [ + 0.9341399501348013, + 0.8204878351747936, + 0.741718257843464, + 0.873138223676903, + 0.9667812319980432, + 0.9508990473624674, + 0.8583715106367068, + 0.8317672039812323 + ], + [ + 0.7693840011034969, + 0.6242272758951368, + 0.7334020420014349, + 0.819758614048956, + 0.8113594410078233, + 0.9287353649003786, + 0.5350819109954088, + 0.8094850373908528 + ], + [ + 0.8724338125781085, + 0.7061961172340323, + 0.6483117848277336, + 0.9270996365242617, + 0.8145149135469096, + 0.6027771739659058, + 0.7267042477336481, + 0.7256106133096523 + ], + [ + 0.697487416139134, + 0.7749642378764864, + 0.7542641966355893, + 0.6534019716956425, + 0.6771193862095888, + 0.5231010396321594, + 0.6619939174844891, + 0.8026450070480431 + ], + [ + 0.7096837249810822, + 0.8327177975080668, + 0.6102276412955541, + 0.6784356503444863, + 0.45911370035656884, + 0.6282525410779394, + 0.7583828320211272, + 0.4331776824319875 + ], + [ + 0.5811465299728954, + 0.7825894189524019, + 0.8520666218959584, + 0.730418314777868, + 0.8115417731985337, + 0.31088769586439957, + 0.7750663378092637, + 0.7552346103096029 + ], + [ + 0.7297139859773298, + 0.8427559551201818, + 0.6560984578427342, + 0.7998189302844246, + 0.8067103992260668, + 0.6542840061923342, + 0.8123960664484118, + 0.6481010838473075 + ] + ], + "n": [ + [ + 5391, + 5391, + 5391, + 5391, + 5391, + 5391, + 5391, + 5391 + ], + [ + 4940, + 5391, + 5391, + 5314, + 4905, + 5391, + 5391, + 5391 + ], + [ + 5390, + 5382, + 5391, + 5391, + 5391, + 5391, + 5391, + 5391 + ], + [ + 5391, + 5391, + 5385, + 5390, + 5391, + 0, + 5387, + 5360 + ], + [ + 5391, + 5367, + 5391, + 5309, + 5307, + 5344, + 5391, + 5372 + ], + [ + 5390, + 5385, + 5391, + 5371, + 5391, + 5382, + 5391, + 5363 + ], + [ + 5265, + 5390, + 5391, + 3857, + 5391, + 5373, + 5390, + 5387 + ], + [ + 5391, + 5251, + 5378, + 5303, + 5385, + 5304, + 5310, + 5391 + ], + [ + 5391, + 5391, + 5391, + 5339, + 5391, + 5390, + 5390, + 5388 + ], + [ + 5391, + 5391, + 5295, + 5391, + 5391, + 5391, + 5376, + 5391 + ], + [ + 5390, + 5370, + 5391, + 5391, + 5374, + 5384, + 5384, + 5391 + ], + [ + 5391, + 5391, + 5386, + 5391, + 5384, + 5391, + 5387, + 5383 + ], + [ + 5391, + 5391, + 5377, + 5390, + 5389, + 5391, + 5391, + 5391 + ], + [ + 5360, + 5391, + 5391, + 5390, + 5267, + 5391, + 5391, + 5375 + ], + [ + 5337, + 5371, + 5377, + 5366, + 5387, + 5238, + 5391, + 5383 + ], + [ + 5340, + 5382, + 5385, + 5391, + 5391, + 5385, + 5391, + 5351 + ] + ] + }, + "WHITESP": { + "gamma": [ + [ + 0.9481587431421652, + 1.1858258308285232, + 0.7215592120737407, + 1.4401303176260352, + 1.8469700365408603, + 1.0690998365102504, + 0.9679893758722993, + 0.8251417102480029 + ], + [ + 1.8336932903644905, + 2.0066504018240092, + 0.6728715400377117, + 2.0680819731876072, + 0.4819994043893982, + 0.5931478858470546, + 0.9527822886461026, + 0.5600586108871585 + ], + [ + 1.2783146869531823, + 0.9940700578782079, + 1.024902139641089, + 0.7313846508848372, + 0.9593716249359724, + 0.8042134863451078, + 1.421264059277694, + 1.261037188155484 + ], + [ + 0.4152854169176438, + 0.37606959606566276, + 2.2141393100232007, + 0.7521941990892569, + 0.8957562200209143, + NaN, + 0.9779208321037696, + 0.35832484260562697 + ], + [ + 0.17699898331691866, + 1.5314355896857668, + 0.8872272005408038, + 0.777241087669203, + 1.4747687660586932, + 1.8365935398863706, + 0.5157948762908545, + 0.6541518304600921 + ], + [ + 0.8381785881294093, + 0.9384809957973652, + 0.7935092864311561, + 1.9170345951997712, + 0.5704649087983481, + 0.612542074708743, + 0.47063508568941437, + 0.35528844335648824 + ], + [ + 0.38373093514145074, + 0.11005709837832675, + 0.8980971258041471, + 0.5794470618334423, + 0.29640118572607355, + 0.9452542843364213, + -0.23629173636189513, + 0.2864715503565068 + ], + [ + 0.5011666319002807, + 0.14145799815444704, + 0.7886538117347673, + 0.9017566407796498, + 1.1302756254255395, + 0.2343799152255389, + 0.7012805172015381, + 1.2301968979997626 + ], + [ + -0.11846690649434462, + 0.6043121815600754, + 0.83556615656461, + 0.5368331853619002, + 0.0352506277880323, + 0.41524501800638147, + 0.7197608085519034, + 0.47244163233667813 + ], + [ + 0.42914300695874663, + 0.42210012023696514, + 1.1110688882410054, + 0.47710214977356863, + 0.4173693456516617, + 0.2849774623421524, + 0.8135697605565675, + 0.619152129714185 + ], + [ + 0.02742063070457024, + 1.167986103085252, + 0.7442656021357891, + 0.7067743143275773, + 1.0596578842437228, + 0.552940840598305, + 0.3483453128059129, + 0.7417335653153394 + ], + [ + 0.4622985352668484, + 0.37756894133735747, + 0.47804638739148214, + 0.6115790832625634, + 0.6585672954581852, + 0.31488206814046904, + 0.5193811929007589, + 0.39212257054346306 + ], + [ + 0.7879694771243151, + 0.47711364258790706, + 0.6387752960242578, + 0.5914954258778918, + 0.8248357417555092, + 0.7046587831602595, + 0.43921431619211254, + 0.29285483116364835 + ], + [ + 1.2509829335050267, + 0.35273666646150853, + 0.7837148268361732, + 0.5627873909639364, + 1.8423670176118658, + 0.5035568216237719, + 0.33595576569496544, + 0.35672325628508206 + ], + [ + 1.8588601153837419, + 1.1605247139393435, + 1.0493759878929154, + 1.4650280950907275, + 1.1088028134113803, + 1.3853690407924621, + 0.42770648776159664, + 1.040381206298663 + ], + [ + 1.1711682665666139, + 0.8418495317430749, + 1.3184848092007848, + 0.8014167508227809, + 0.4731105883165524, + 0.9545925353207952, + 0.4224493249294723, + 0.9976542893946937 + ] + ], + "r2": [ + [ + 0.9601799242470082, + 0.8983250157691647, + 0.9492039347836537, + 0.9274323501563004, + 0.8481635456376309, + 0.9379768962510217, + 0.9514212243321898, + 0.9584371125460295 + ], + [ + 0.7575063584711861, + 0.9938455042055736, + 0.9078956976845586, + 0.7805241000778329, + 0.20718206454770394, + 0.9021571135626346, + 0.8266580196905335, + 0.6683095004227353 + ], + [ + 0.8269415752779956, + 0.6246169856668933, + 0.6809142430220396, + 0.7807105262095756, + 0.6165934686937429, + 0.6127200125802945, + 0.9412937996515022, + 0.8359876559709568 + ], + [ + 0.6205529405399999, + 0.5159308527391093, + 0.9284601717673011, + 0.7513214791977649, + 0.9154176530531394, + 0.0, + 0.7861091946944054, + 0.5122433245618528 + ], + [ + 0.1661272873757661, + 0.5885351006119544, + 0.750712484104379, + 0.4344914104533053, + 0.7054482819036252, + 0.729411532671476, + 0.8630754244013719, + 0.313932908730011 + ], + [ + 0.6616015633535431, + 0.521184920960557, + 0.7420524438382843, + 0.6704102301283962, + 0.4811290578592453, + 0.4103255794976368, + 0.32289423131357275, + 0.3393943313361719 + ], + [ + 0.052647223706414614, + 0.0821756819792504, + 0.7645675004532145, + 0.3359593102427243, + 0.28206102531378974, + 0.7754390465113523, + 0.4741947882708718, + 0.4461009610693806 + ], + [ + 0.7320844451769182, + 0.020047410071668215, + 0.6357162152230647, + 0.567466756590899, + 0.8769633659132552, + 0.025021572002058234, + 0.12334397113971907, + 0.726859616271307 + ], + [ + 0.16618074077290435, + 0.801612600153647, + 0.8499947236121186, + 0.5165365000764024, + 0.0018393843892064998, + 0.46493736137334374, + 0.8486421753451558, + 0.3428262452823072 + ], + [ + 0.6939773828532829, + 0.8386221707461912, + 0.2755618374778185, + 0.4735443771965019, + 0.889502187028302, + 0.47850267236243205, + 0.4504636952709229, + 0.9063103063651692 + ], + [ + 0.018043439243578874, + 0.5134157659656153, + 0.8576707017589917, + 0.7446109883506021, + 0.9315896067164214, + 0.6608882684032258, + 0.5890340246599806, + 0.5674151022751663 + ], + [ + 0.59221171150559, + 0.7149534803759545, + 0.4448858554576466, + 0.8513139127533543, + 0.7779977868497705, + 0.5359499211732865, + 0.5505938660906109, + 0.78048859005064 + ], + [ + 0.5438825086257759, + 0.49914241112614177, + 0.6921191803076903, + 0.5694972535322367, + 0.5488490947079256, + 0.5292779116511044, + 0.38707892030039626, + 0.24936192251834588 + ], + [ + 0.25057524703921175, + 0.47906692087670966, + 0.6164883176679207, + 0.6672352315629575, + 0.5472222967173989, + 0.5223573631301592, + 0.49497650179947217, + 0.34136931716813956 + ], + [ + 0.5369937638951057, + 0.6714997059258943, + 0.6839596767146889, + 0.6500021932474407, + 0.6143178293878929, + 0.5668094174513334, + 0.8538855833444818, + 0.34276417601319686 + ], + [ + 0.588288765465703, + 0.9064465496606747, + 0.5152175059241366, + 0.7953242510089032, + 0.872378754516979, + 0.7862408858398378, + 0.7337841302910681, + 0.7657134273095694 + ] + ], + "n": [ + [ + 1601, + 1601, + 1601, + 1601, + 1601, + 1601, + 1601, + 1601 + ], + [ + 1593, + 1601, + 1601, + 1599, + 1392, + 1601, + 1601, + 1601 + ], + [ + 1601, + 1601, + 1601, + 1601, + 1601, + 1601, + 1601, + 1601 + ], + [ + 1601, + 1601, + 1601, + 1601, + 1601, + 27, + 1601, + 1601 + ], + [ + 1601, + 1599, + 1601, + 1598, + 1555, + 1601, + 1601, + 1595 + ], + [ + 1601, + 1601, + 1601, + 1600, + 1601, + 1601, + 1601, + 1601 + ], + [ + 1601, + 1601, + 1601, + 1505, + 1601, + 1601, + 1601, + 1601 + ], + [ + 1601, + 1597, + 1599, + 1600, + 1600, + 1596, + 1598, + 1601 + ], + [ + 1601, + 1601, + 1601, + 1601, + 1600, + 1597, + 1601, + 1601 + ], + [ + 1601, + 1601, + 1587, + 1600, + 1601, + 1601, + 1601, + 1601 + ], + [ + 1601, + 1600, + 1601, + 1601, + 1595, + 1601, + 1599, + 1601 + ], + [ + 1601, + 1601, + 1601, + 1601, + 1601, + 1601, + 1601, + 1601 + ], + [ + 1601, + 1601, + 1601, + 1601, + 1601, + 1601, + 1601, + 1601 + ], + [ + 1592, + 1601, + 1601, + 1601, + 1595, + 1601, + 1601, + 1601 + ], + [ + 1593, + 1600, + 1600, + 1600, + 1601, + 1595, + 1601, + 1597 + ], + [ + 1601, + 1601, + 1601, + 1601, + 1601, + 1601, + 1601, + 1601 + ] + ] + }, + "MID": { + "gamma": [ + [ + 0.9538982645762746, + 1.1004128305104712, + 0.8590508197841705, + 1.3400965191775198, + 2.131354705878869, + 1.2784094050594221, + 0.7951988324938795, + 1.1604128826540558 + ], + [ + 1.3186755451719345, + 1.9789438626302729, + 0.5641762256529401, + 2.2267381518209572, + 0.6752016139182333, + 0.5461112702044442, + 0.8577659535632407, + 0.785045556209174 + ], + [ + 0.9721731335799588, + 1.0961159423588247, + 0.7911682023375262, + 0.5697056749489628, + 0.7044091849535137, + 0.9981532360668234, + 0.6511505208150729, + 0.9015303213734919 + ], + [ + 0.5050561259912645, + 0.9901793646582339, + 1.2706771260534249, + 0.5539558765404042, + 0.6789854675246654, + -0.056250518193965225, + 0.8796156397133232, + 0.3962612886008694 + ], + [ + 0.3208304403285207, + -0.1526476119436985, + 0.050486414429011715, + -0.3078557924263328, + -0.17425398481946822, + -0.3015388781716261, + 0.11446457712157898, + -0.17058448891976719 + ], + [ + 0.11655677516243339, + 0.02026345381320935, + 0.10983247895938786, + -0.36877916256969934, + 0.1062378977203378, + -0.690631383990702, + -0.11671345762611915, + 0.03178590853278816 + ], + [ + -0.549081726896633, + 0.09019753478884411, + 0.18426429872602257, + -0.048385500382743474, + 0.19432430354543354, + -0.01791010742653547, + 0.22005901418750812, + -0.0017222242169615306 + ], + [ + -0.15510366486975774, + -0.29567259418935027, + -0.2582022953719935, + -0.5281034240115955, + 1.4047037844636203, + -0.5041596421727638, + -0.5382723260651572, + 0.11589571136975699 + ], + [ + -0.1462076098908322, + 0.1712124083749214, + 0.25577768837618414, + 0.10392636748278627, + 0.0804828372640174, + 0.029986240622834757, + 0.15542535294451568, + -0.05336058573227025 + ], + [ + 0.18029990527221237, + -0.014898869882557845, + -0.30272828671507135, + 0.2244115189934465, + 0.10876203024786818, + 0.18020957221389464, + 0.09422480135240126, + 0.2648113673086382 + ], + [ + -0.19238625165735418, + -0.39963366899342306, + 0.056473226681513354, + 0.0467491042000442, + -0.005220574881571431, + -0.13119382547667136, + -0.24325820776299786, + 0.05152409295094109 + ], + [ + 0.11879562086614089, + -0.10430273175110497, + -0.18560928593870574, + 4.5902547981251096e-05, + 0.019292012791576614, + -0.19517707688311284, + -0.17576672752763464, + -0.05360835942296224 + ], + [ + -0.12665637372346714, + 0.0074526867393460506, + -0.24022900434821662, + -0.06558400605600402, + -0.21876341800190294, + -0.27302932895268456, + -0.10181985846707531, + 0.05387762856504629 + ], + [ + -0.3553911436506451, + 0.02434125460413231, + -0.17523111435871022, + -0.12691518293200935, + -0.5742109161851909, + -0.2397968158881883, + 0.01102752978188021, + -0.4669626408514464 + ], + [ + -0.48546787720247225, + -0.19855915926206233, + 0.10242460812314737, + -0.43316658932513785, + -0.023586835455956565, + -0.4841549478753385, + 0.013134530588882809, + -0.0672427359288492 + ], + [ + -0.3491147269385385, + -0.08731053331342313, + -0.298345195697989, + -0.003912489722910926, + 0.003985741318285619, + -0.26342101738231055, + 0.022600988889820567, + -0.1641736967293374 + ] + ], + "r2": [ + [ + 0.9686350690102418, + 0.9932089030230603, + 0.8596923254809096, + 0.9341175715346031, + 0.9891004249087142, + 0.9739101293596282, + 0.9793529016443954, + 0.9672094241951024 + ], + [ + 0.4469307109546361, + 0.9834774370794305, + 0.9013905965797753, + 0.8054902569522526, + 0.2974230407499179, + 0.9050819046003891, + 0.8141817172136545, + 0.9821941760165281 + ], + [ + 0.9530782096637087, + 0.5144674855157159, + 0.7795194722346226, + 0.45700404815257656, + 0.978906026813552, + 0.7127483948578178, + 0.782037360844163, + 0.9978980137731566 + ], + [ + 0.9637954230668613, + 0.9827678755200894, + 0.8901895002933757, + 0.7245077103522806, + 0.7512828394902631, + 0.011742223078264336, + 0.70682033163811, + 0.3351569018739793 + ], + [ + 0.4688989681207669, + 0.09228750783443662, + 0.0237134939324416, + 0.6349612303639348, + 0.3637562763784187, + 0.42860606049155914, + 0.06451820913980877, + 0.11669498099333098 + ], + [ + 0.04841324849482209, + 0.00881316236727503, + 0.14910295132690088, + 0.5091128210120198, + 0.10161405665206447, + 0.8389043603586731, + 0.22056084946389098, + 0.006402044468875268 + ], + [ + 0.802292865096805, + 0.06148591640880963, + 0.17939687784337077, + 0.009636729068616345, + 0.16156253949927257, + 0.0012621794353719906, + 0.28409663315740386, + 3.251601083675837e-05 + ], + [ + 0.36213438855866, + 0.6607177024623769, + 0.5169940291975987, + 0.7683819200910538, + 0.9005618596412314, + 0.7156970574420319, + 0.734747558650114, + 0.06560576447283373 + ], + [ + 0.25193338676980437, + 0.22204033206042895, + 0.41661310386422845, + 0.04649810048391767, + 0.030263276768977998, + 0.009732797181104091, + 0.1606214927063513, + 0.016549352792561978 + ], + [ + 0.5015909298769199, + 0.00165454686965405, + 0.31554713174770277, + 0.16601351866472858, + 0.147218319845141, + 0.5874958866455018, + 0.03618666464128051, + 0.33128144869768783 + ], + [ + 0.2769690610700982, + 0.6020861990709792, + 0.03661087935287621, + 0.012606007091435667, + 0.0001222734999689612, + 0.20503529109108554, + 0.5398229357500206, + 0.016208642402548268 + ], + [ + 0.11004716343842191, + 0.09715921077802858, + 0.2250126781135635, + 2.081040562096348e-08, + 0.0018193208747898648, + 0.31995418558634026, + 0.3089934357680082, + 0.035780511235440016 + ], + [ + 0.15983095573149642, + 0.00037397716189957464, + 0.294362620556086, + 0.03853365375436946, + 0.24508069560718337, + 0.4494629248291365, + 0.1276243627937811, + 0.047500374002230505 + ], + [ + 0.39587456449381353, + 0.005898296429852512, + 0.2602283680313119, + 0.15538250395784958, + 0.7964793051769032, + 0.33046759847884466, + 0.0019858739792645297, + 0.6373626709909426 + ], + [ + 0.5908368815150173, + 0.21312715888732214, + 0.05995949317381832, + 0.6231536190733793, + 0.0028237821640131333, + 0.7541961499337564, + 0.0024226719980128264, + 0.017517509437273593 + ], + [ + 0.502068958772049, + 0.041459394237980285, + 0.3624293901539417, + 0.0001664644373576829, + 0.00013801278882963341, + 0.3210437248315625, + 0.005170446201017587, + 0.15256503079462025 + ] + ], + "n": [ + [ + 94640, + 94635, + 94640, + 94614, + 94640, + 94640, + 94640, + 94630 + ], + [ + 87636, + 94640, + 94630, + 93246, + 86875, + 94640, + 94640, + 94640 + ], + [ + 94603, + 94539, + 94639, + 94637, + 94640, + 94640, + 94640, + 94640 + ], + [ + 94640, + 94640, + 94606, + 94637, + 94640, + 99, + 93826, + 94599 + ], + [ + 94636, + 94091, + 94640, + 92356, + 93186, + 94362, + 94640, + 94025 + ], + [ + 94554, + 94580, + 94640, + 94344, + 94640, + 94261, + 94639, + 93980 + ], + [ + 92582, + 94623, + 94640, + 85305, + 94633, + 94549, + 94606, + 94590 + ], + [ + 94569, + 93059, + 94625, + 93360, + 94503, + 92412, + 93385, + 94640 + ], + [ + 94638, + 94640, + 94640, + 94392, + 94609, + 94607, + 94626, + 94598 + ], + [ + 94640, + 94639, + 93002, + 94550, + 94640, + 94637, + 94189, + 94598 + ], + [ + 94634, + 94492, + 94640, + 94637, + 94510, + 94614, + 94517, + 94640 + ], + [ + 94639, + 94618, + 94616, + 94640, + 94525, + 94634, + 94632, + 94585 + ], + [ + 94636, + 94638, + 94456, + 94636, + 94615, + 94640, + 94620, + 94631 + ], + [ + 94246, + 94640, + 94639, + 94617, + 93166, + 94640, + 94638, + 94238 + ], + [ + 93634, + 94134, + 94378, + 94076, + 94366, + 91748, + 94638, + 94429 + ], + [ + 94118, + 94466, + 94557, + 94581, + 94640, + 94550, + 94636, + 93883 + ] + ] + } + } +} \ No newline at end of file diff --git a/data/exp_multifractal/EleutherAI--pythia-2.8b_multifractal_v2.json b/data/exp_multifractal/EleutherAI--pythia-2.8b_multifractal_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..9995ef2d4655156eef884a40883f27c8bdbcd32e --- /dev/null +++ b/data/exp_multifractal/EleutherAI--pythia-2.8b_multifractal_v2.json @@ -0,0 +1,51 @@ +{ + "model": "EleutherAI/pythia-2.8b", + "n_layers": 32, + "n_heads": 32, + "n_prompts": 50, + "seq_len": 1100, + "summary": { + "PUNCT": { + "median_gamma_good": 0.2856476458860821, + "iqr_gamma_good": 1.2096973106029958, + "n_good": 300, + "n_total": 1024, + "mean_r2": 0.30846221533659146 + }, + "COMMON": { + "median_gamma_good": -0.421589187258973, + "iqr_gamma_good": 1.3949766244469055, + "n_good": 242, + "n_total": 1024, + "mean_r2": 0.27288483897702187 + }, + "RARE": { + "median_gamma_good": 0.5851144126693855, + "iqr_gamma_good": 1.4518170241225195, + "n_good": 148, + "n_total": 1024, + "mean_r2": 0.22978847887629006 + }, + "NUMERIC": { + "median_gamma_good": 0.8695919211258053, + "iqr_gamma_good": 0.42385599184562506, + "n_good": 614, + "n_total": 1024, + "mean_r2": 0.5189374455325683 + }, + "WHITESP": { + "median_gamma_good": 0.8481436473418742, + "iqr_gamma_good": 0.5575323551567233, + "n_good": 613, + "n_total": 1024, + "mean_r2": 0.497662638153801 + }, + "MID": { + "median_gamma_good": -0.38145584725038295, + "iqr_gamma_good": 1.0269605167046376, + "n_good": 334, + "n_total": 1024, + "mean_r2": 0.3412117810330888 + } + } +} \ No newline at end of file diff --git a/data/exp_multifractal/EleutherAI--pythia-410m_multifractal_v2.json b/data/exp_multifractal/EleutherAI--pythia-410m_multifractal_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..665de1dcb874a3e39438216f286382705afe686d --- /dev/null +++ b/data/exp_multifractal/EleutherAI--pythia-410m_multifractal_v2.json @@ -0,0 +1,51 @@ +{ + "model": "EleutherAI/pythia-410m", + "n_layers": 24, + "n_heads": 16, + "n_prompts": 50, + "seq_len": 1100, + "summary": { + "PUNCT": { + "median_gamma_good": 0.6471152420930161, + "iqr_gamma_good": 0.7752038815018629, + "n_good": 117, + "n_total": 384, + "mean_r2": 0.30779052735322004 + }, + "COMMON": { + "median_gamma_good": 0.48246049699427795, + "iqr_gamma_good": 1.2593116713055694, + "n_good": 124, + "n_total": 384, + "mean_r2": 0.31965237030160826 + }, + "RARE": { + "median_gamma_good": 0.48119784177106495, + "iqr_gamma_good": 0.7962303439821159, + "n_good": 100, + "n_total": 384, + "mean_r2": 0.2744249750329164 + }, + "NUMERIC": { + "median_gamma_good": 0.8835322569518108, + "iqr_gamma_good": 0.6433614891327555, + "n_good": 203, + "n_total": 384, + "mean_r2": 0.45398765243473643 + }, + "WHITESP": { + "median_gamma_good": 0.9198389658586196, + "iqr_gamma_good": 0.8831242473914834, + "n_good": 121, + "n_total": 384, + "mean_r2": 0.29721821521638897 + }, + "MID": { + "median_gamma_good": 0.39900538723515167, + "iqr_gamma_good": 1.1631052622395162, + "n_good": 139, + "n_total": 384, + "mean_r2": 0.33794021885753356 + } + } +} \ No newline at end of file diff --git a/data/exp_multifractal/HuggingFaceTB--SmolLM2-360M_multifractal_v2.json b/data/exp_multifractal/HuggingFaceTB--SmolLM2-360M_multifractal_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..9490c0d080ce89786eb1fc846b6403ad164e85f9 --- /dev/null +++ b/data/exp_multifractal/HuggingFaceTB--SmolLM2-360M_multifractal_v2.json @@ -0,0 +1,51 @@ +{ + "model": "HuggingFaceTB/SmolLM2-360M", + "n_layers": 32, + "n_heads": 15, + "n_prompts": 50, + "seq_len": 1100, + "summary": { + "PUNCT": { + "median_gamma_good": -0.4325112533097145, + "iqr_gamma_good": 1.168201212449327, + "n_good": 152, + "n_total": 480, + "mean_r2": 0.3393576073114089 + }, + "COMMON": { + "median_gamma_good": -0.5760268523756398, + "iqr_gamma_good": 1.1112376459640803, + "n_good": 215, + "n_total": 480, + "mean_r2": 0.4401709002687807 + }, + "RARE": { + "median_gamma_good": -0.64061253357996, + "iqr_gamma_good": 0.502848377121444, + "n_good": 291, + "n_total": 480, + "mean_r2": 0.5517666041463135 + }, + "NUMERIC": { + "median_gamma_good": 0.4113149677375936, + "iqr_gamma_good": 0.9948692088567266, + "n_good": 188, + "n_total": 480, + "mean_r2": 0.41452813335138017 + }, + "WHITESP": { + "median_gamma_good": -0.2644434201267102, + "iqr_gamma_good": 1.1282395139101955, + "n_good": 291, + "n_total": 480, + "mean_r2": 0.5706900182292782 + }, + "MID": { + "median_gamma_good": -0.5957919059178234, + "iqr_gamma_good": 0.45875801239848907, + "n_good": 236, + "n_total": 480, + "mean_r2": 0.4666905136170288 + } + } +} \ No newline at end of file diff --git a/data/exp_multifractal/Qwen--Qwen2.5-1.5B_multifractal_v2.json b/data/exp_multifractal/Qwen--Qwen2.5-1.5B_multifractal_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..495bb303a43eb3055e0484a9f50d8c742471d05e --- /dev/null +++ b/data/exp_multifractal/Qwen--Qwen2.5-1.5B_multifractal_v2.json @@ -0,0 +1,51 @@ +{ + "model": "Qwen/Qwen2.5-1.5B", + "n_layers": 28, + "n_heads": 12, + "n_prompts": 50, + "seq_len": 1100, + "summary": { + "PUNCT": { + "median_gamma_good": 1.215579508020458, + "iqr_gamma_good": 0.6466833055656311, + "n_good": 8, + "n_total": 336, + "mean_r2": 0.02054025828829171 + }, + "COMMON": { + "median_gamma_good": 1.2082695466428857, + "iqr_gamma_good": 0.3932278114506753, + "n_good": 9, + "n_total": 336, + "mean_r2": 0.021877626107433806 + }, + "RARE": { + "median_gamma_good": 0.9510497279589039, + "iqr_gamma_good": 0.1719710517541122, + "n_good": 5, + "n_total": 336, + "mean_r2": 0.014819719561501991 + }, + "NUMERIC": { + "median_gamma_good": 1.0281647855294356, + "iqr_gamma_good": 0.386727920107959, + "n_good": 8, + "n_total": 336, + "mean_r2": 0.02101423541081187 + }, + "WHITESP": { + "median_gamma_good": 1.2196060139456133, + "iqr_gamma_good": 0.9328794502235627, + "n_good": 8, + "n_total": 336, + "mean_r2": 0.02054414224176208 + }, + "MID": { + "median_gamma_good": 1.0393963689057721, + "iqr_gamma_good": 0.1720815905751275, + "n_good": 6, + "n_total": 336, + "mean_r2": 0.019486935554480445 + } + } +} \ No newline at end of file diff --git a/data/exp_neural_thermostat/results.json b/data/exp_neural_thermostat/results.json new file mode 100644 index 0000000000000000000000000000000000000000..de18a959e71a6c438d0d0f4232ac783ca2479fc7 --- /dev/null +++ b/data/exp_neural_thermostat/results.json @@ -0,0 +1,28 @@ +{ + "model": "EleutherAI/pythia-1b", + "n_chunks": 50, + "baseline_ppl": 15.714571287245102, + "results": { + "baseline_no_hook": { + "ppl": 15.714571287245102 + }, + "identity": { + "ppl": 15.723431903954907, + "delta_vs_baseline": 0.008860616709805313, + "mean_nll": 2.755152077674866, + "last_gate_mean": 0.9933071732521057 + }, + "aggressive_mute": { + "ppl": 4013.370282414504, + "delta_vs_baseline": 3997.6557111272587, + "mean_nll": 8.297386636734009, + "last_gate_mean": 0.006692850962281227 + }, + "expert_prior": { + "ppl": 15.723431903954907, + "delta_vs_baseline": 0.008860616709805313, + "mean_nll": 2.755152077674866, + "last_gate_mean": 0.9933071732521057 + } + } +} \ No newline at end of file diff --git a/data/exp_neural_thermostat_v2/results.json b/data/exp_neural_thermostat_v2/results.json new file mode 100644 index 0000000000000000000000000000000000000000..6da858a41ab1fcf69c6a9a8c050efd14648789e1 --- /dev/null +++ b/data/exp_neural_thermostat_v2/results.json @@ -0,0 +1,35 @@ +{ + "model": "EleutherAI/pythia-1b", + "n_chunks": 50, + "baseline_ppl": 15.714571287245102, + "results": { + "baseline": { + "ppl": 15.714571287245102 + }, + "identity": { + "ppl": 15.714571287245102, + "delta": 0.0, + "mean_gate": 1.0 + }, + "cool_top25_entropy": { + "ppl": 18.087363181636267, + "delta": 2.372791894391165, + "mean_gate": 0.875 + }, + "cool_top25_gamma": { + "ppl": 17.373013858379643, + "delta": 1.6584425711345414, + "mean_gate": 0.875 + }, + "cool_late_layers": { + "ppl": 20.825448372378585, + "delta": 5.110877085133483, + "mean_gate": 1.0 + }, + "cool_early_layers": { + "ppl": 84.632623816721, + "delta": 68.91805252947589, + "mean_gate": 0.5 + } + } +} \ No newline at end of file diff --git a/data/exp_prune/EleutherAI--pythia-1b_prune.json b/data/exp_prune/EleutherAI--pythia-1b_prune.json new file mode 100644 index 0000000000000000000000000000000000000000..d624d1d5634ffaf6b849beb6ee3c6ab9807bb28f --- /dev/null +++ b/data/exp_prune/EleutherAI--pythia-1b_prune.json @@ -0,0 +1,74 @@ +{ + "model": "EleutherAI/pythia-1b", + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "n_pairs": 128, + "k_dead": 91, + "L_crit_pred": 15, + "alpha": 0.9374717358310781, + "n_eval_chunks": 100, + "chunk_size": 1024, + "baseline": { + "ppl": 14.973609132474365, + "nll": 2.7062892603874205, + "std": 0.2595318035858822 + }, + "sweeps": [ + { + "L_prune": 13, + "n_pruned_entries": 7274496, + "frac_qk_pruned": 0.05419921875, + "ppl": 15.548956277713987, + "nll": 2.743993515968323, + "std": 0.25814710014606357, + "delta_ppl": 0.5753471452396219, + "delta_nll": 0.0377042555809024, + "verdict": "DEGRADED" + }, + { + "L_prune": 15, + "n_pruned_entries": 2424832, + "frac_qk_pruned": 0.01806640625, + "ppl": 15.16368330360712, + "nll": 2.7189033126831053, + "std": 0.2584823516819064, + "delta_ppl": 0.19007417113275515, + "delta_nll": 0.012614052295684797, + "verdict": "OK" + }, + { + "L_prune": 16, + "n_pruned_entries": 0, + "frac_qk_pruned": 0.0, + "ppl": 14.973609132474365, + "nll": 2.7062892603874205, + "std": 0.2595318035858822, + "delta_ppl": 0.0, + "delta_nll": 0.0, + "verdict": "OK" + }, + { + "L_prune": 17, + "n_pruned_entries": 0, + "frac_qk_pruned": 0.0, + "ppl": 14.973609132474365, + "nll": 2.7062892603874205, + "std": 0.2595318035858822, + "delta_ppl": 0.0, + "delta_nll": 0.0, + "verdict": "OK" + }, + { + "L_prune": 19, + "n_pruned_entries": 0, + "frac_qk_pruned": 0.0, + "ppl": 14.973609132474365, + "nll": 2.7062892603874205, + "std": 0.2595318035858822, + "delta_ppl": 0.0, + "delta_nll": 0.0, + "verdict": "OK" + } + ] +} \ No newline at end of file diff --git a/data/exp_prune/EleutherAI--pythia-1b_prune_v2.json b/data/exp_prune/EleutherAI--pythia-1b_prune_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..d181e27fdb20749f7788922938bc16e15924def2 --- /dev/null +++ b/data/exp_prune/EleutherAI--pythia-1b_prune_v2.json @@ -0,0 +1,75 @@ +{ + "model": "EleutherAI/pythia-1b", + "n_layers": 16, + "n_heads": 8, + "d_head": 256, + "n_pairs": 128, + "k_dead": 91, + "theta": 10000.0, + "T_train": 2048, + "L_crit_pred": 15, + "alpha": 0.9374717358310781, + "n_eval_chunks": 100, + "chunk_size": 1024, + "n_lambada": 200, + "baseline": { + "ppl": 14.973609132474365, + "nll": 2.7062892603874205, + "std": 0.2595318035858822, + "lambada_acc": 0.57 + }, + "sweeps": [ + { + "L_prune": 11, + "n_pruned_entries": 12124160, + "frac_qk_pruned": 0.09033203125, + "ppl": 16.53646806228249, + "nll": 2.805568127632141, + "std": 0.2573620626824769, + "delta_ppl": 1.562858929808126, + "delta_nll": 0.09927886724472046, + "lambada_acc": 0.405, + "delta_lambada": -0.16499999999999992, + "verdict": "DEGRADED" + }, + { + "L_prune": 13, + "n_pruned_entries": 7274496, + "frac_qk_pruned": 0.05419921875, + "ppl": 15.548956277713987, + "nll": 2.743993515968323, + "std": 0.25814710014606357, + "delta_ppl": 0.5753471452396219, + "delta_nll": 0.0377042555809024, + "lambada_acc": 0.55, + "delta_lambada": -0.019999999999999907, + "verdict": "DEGRADED" + }, + { + "L_prune": 15, + "n_pruned_entries": 2424832, + "frac_qk_pruned": 0.01806640625, + "ppl": 15.16368330360712, + "nll": 2.7189033126831053, + "std": 0.2584823516819064, + "delta_ppl": 0.19007417113275515, + "delta_nll": 0.012614052295684797, + "lambada_acc": 0.54, + "delta_lambada": -0.029999999999999916, + "verdict": "OK" + }, + { + "L_prune": 16, + "n_pruned_entries": 0, + "frac_qk_pruned": 0.0, + "ppl": 14.973609132474365, + "nll": 2.7062892603874205, + "std": 0.2595318035858822, + "delta_ppl": 0.0, + "delta_nll": 0.0, + "lambada_acc": 0.57, + "delta_lambada": 0.0, + "verdict": "OK" + } + ] +} \ No newline at end of file diff --git a/data/exp_prune/EleutherAI--pythia-2.8b_prune_v2.json b/data/exp_prune/EleutherAI--pythia-2.8b_prune_v2.json new file mode 100644 index 0000000000000000000000000000000000000000..f14ddbe2df16274209d392437c97a9974a9e97c5 --- /dev/null +++ b/data/exp_prune/EleutherAI--pythia-2.8b_prune_v2.json @@ -0,0 +1,75 @@ +{ + "model": "EleutherAI/pythia-2.8b", + "n_layers": 32, + "n_heads": 32, + "d_head": 80, + "n_pairs": 40, + "k_dead": 29, + "theta": 10000.0, + "T_train": 2048, + "L_crit_pred": 21, + "alpha": 0.6700912468018047, + "n_eval_chunks": 100, + "chunk_size": 1024, + "n_lambada": 200, + "baseline": { + "ppl": 11.497348191010678, + "nll": 2.4421164166927336, + "std": 0.2581671149222198, + "lambada_acc": 0.615 + }, + "sweeps": [ + { + "L_prune": 17, + "n_pruned_entries": 54067200, + "frac_qk_pruned": 0.12890625, + "ppl": 14.078933084106032, + "nll": 2.6446795725822447, + "std": 0.2594730613347292, + "delta_ppl": 2.5815848930953536, + "delta_nll": 0.20256315588951113, + "lambada_acc": 0.425, + "delta_lambada": -0.19, + "verdict": "DEGRADED" + }, + { + "L_prune": 19, + "n_pruned_entries": 46858240, + "frac_qk_pruned": 0.11171875, + "ppl": 13.294295816582924, + "nll": 2.587335057258606, + "std": 0.2611973881683509, + "delta_ppl": 1.7969476255722459, + "delta_nll": 0.1452186405658722, + "lambada_acc": 0.485, + "delta_lambada": -0.13, + "verdict": "DEGRADED" + }, + { + "L_prune": 21, + "n_pruned_entries": 39649280, + "frac_qk_pruned": 0.09453125, + "ppl": 12.887260668117273, + "nll": 2.556239278316498, + "std": 0.26105946628430654, + "delta_ppl": 1.389912477106595, + "delta_nll": 0.1141228616237644, + "lambada_acc": 0.475, + "delta_lambada": -0.14, + "verdict": "DEGRADED" + }, + { + "L_prune": 23, + "n_pruned_entries": 32440320, + "frac_qk_pruned": 0.07734375, + "ppl": 12.418053153651558, + "nll": 2.519151313304901, + "std": 0.2594559840569815, + "delta_ppl": 0.9207049626408796, + "delta_nll": 0.07703489661216745, + "lambada_acc": 0.53, + "delta_lambada": -0.08499999999999996, + "verdict": "DEGRADED" + } + ] +} \ No newline at end of file diff --git a/data/exp_t1/checkpoint_results.json b/data/exp_t1/checkpoint_results.json new file mode 100644 index 0000000000000000000000000000000000000000..1772c9031f2dde8912342db369d9ae76cbbe8b9d --- /dev/null +++ b/data/exp_t1/checkpoint_results.json @@ -0,0 +1,490 @@ +{ + "step512": { + "revision": "step512", + "h3": { + "recovery": [ + 0.0, + 0.1148, + 0.3847, + 0.4122, + 0.5515, + 0.7953 + ], + "L_crit_90": null, + "L_crit_99": null, + "mu_baseline": -13.8133, + "mu_d10": -13.4207, + "denom": 0.3927, + "n_prompts": 50 + }, + "wo_rank": { + "layers": [ + { + "layer": 0, + "rank90": 259, + "rank90_frac": 0.5059, + "fro_norm": 7.4594 + }, + { + "layer": 1, + "rank90": 259, + "rank90_frac": 0.5059, + "fro_norm": 7.5453 + }, + { + "layer": 2, + "rank90": 258, + "rank90_frac": 0.5039, + "fro_norm": 7.8337 + }, + { + "layer": 3, + "rank90": 259, + "rank90_frac": 0.5059, + "fro_norm": 7.7319 + }, + { + "layer": 4, + "rank90": 260, + "rank90_frac": 0.5078, + "fro_norm": 7.6855 + }, + { + "layer": 5, + "rank90": 257, + "rank90_frac": 0.502, + "fro_norm": 7.7839 + } + ], + "mean_rank90": 258.67 + }, + "load_time": 3.0 + }, + "step1000": { + "revision": "step1000", + "h3": { + "recovery": [ + 0.0, + 9.1507, + 17.58, + -31.4381, + 16.3582, + -38.5978 + ], + "L_crit_90": 1, + "L_crit_99": 1, + "mu_baseline": -14.3819, + "mu_d10": -14.3795, + "denom": 0.0024, + "n_prompts": 50 + }, + "wo_rank": { + "layers": [ + { + "layer": 0, + "rank90": 243, + "rank90_frac": 0.4746, + "fro_norm": 6.9889 + }, + { + "layer": 1, + "rank90": 248, + "rank90_frac": 0.4844, + "fro_norm": 7.512 + }, + { + "layer": 2, + "rank90": 238, + "rank90_frac": 0.4648, + "fro_norm": 8.3055 + }, + { + "layer": 3, + "rank90": 232, + "rank90_frac": 0.4531, + "fro_norm": 9.7956 + }, + { + "layer": 4, + "rank90": 252, + "rank90_frac": 0.4922, + "fro_norm": 8.4148 + }, + { + "layer": 5, + "rank90": 250, + "rank90_frac": 0.4883, + "fro_norm": 8.1698 + } + ], + "mean_rank90": 243.83 + }, + "load_time": 2.9 + }, + "step2000": { + "revision": "step2000", + "h3": { + "recovery": [ + -0.0, + -0.0209, + 0.0046, + 0.2466, + 0.5994, + 0.7305 + ], + "L_crit_90": null, + "L_crit_99": null, + "mu_baseline": -12.934, + "mu_d10": -14.008, + "denom": -1.074, + "n_prompts": 50 + }, + "wo_rank": { + "layers": [ + { + "layer": 0, + "rank90": 206, + "rank90_frac": 0.4023, + "fro_norm": 8.746 + }, + { + "layer": 1, + "rank90": 221, + "rank90_frac": 0.4316, + "fro_norm": 10.1861 + }, + { + "layer": 2, + "rank90": 201, + "rank90_frac": 0.3926, + "fro_norm": 10.7152 + }, + { + "layer": 3, + "rank90": 223, + "rank90_frac": 0.4355, + "fro_norm": 14.0474 + }, + { + "layer": 4, + "rank90": 226, + "rank90_frac": 0.4414, + "fro_norm": 13.1378 + }, + { + "layer": 5, + "rank90": 233, + "rank90_frac": 0.4551, + "fro_norm": 10.8229 + } + ], + "mean_rank90": 218.33 + }, + "load_time": 4.8 + }, + "step3000": { + "revision": "step3000", + "h3": { + "recovery": [ + -0.0, + 0.0081, + 0.1936, + 0.3893, + 0.3923, + 0.6119 + ], + "L_crit_90": null, + "L_crit_99": null, + "mu_baseline": -12.6007, + "mu_d10": -13.3391, + "denom": -0.7385, + "n_prompts": 50 + }, + "wo_rank": { + "layers": [ + { + "layer": 0, + "rank90": 198, + "rank90_frac": 0.3867, + "fro_norm": 10.641 + }, + { + "layer": 1, + "rank90": 220, + "rank90_frac": 0.4297, + "fro_norm": 12.3794 + }, + { + "layer": 2, + "rank90": 192, + "rank90_frac": 0.375, + "fro_norm": 12.7894 + }, + { + "layer": 3, + "rank90": 230, + "rank90_frac": 0.4492, + "fro_norm": 16.7001 + }, + { + "layer": 4, + "rank90": 231, + "rank90_frac": 0.4512, + "fro_norm": 17.2439 + }, + { + "layer": 5, + "rank90": 238, + "rank90_frac": 0.4648, + "fro_norm": 13.5513 + } + ], + "mean_rank90": 218.17 + }, + "load_time": 13.7 + }, + "step8000": { + "revision": "step8000", + "h3": { + "recovery": [ + -0.0, + -0.001, + 0.163, + 0.4596, + 0.467, + 0.6935 + ], + "L_crit_90": null, + "L_crit_99": null, + "mu_baseline": -13.3228, + "mu_d10": -14.2469, + "denom": -0.9241, + "n_prompts": 50 + }, + "wo_rank": { + "layers": [ + { + "layer": 0, + "rank90": 192, + "rank90_frac": 0.375, + "fro_norm": 14.8205 + }, + { + "layer": 1, + "rank90": 226, + "rank90_frac": 0.4414, + "fro_norm": 16.8638 + }, + { + "layer": 2, + "rank90": 185, + "rank90_frac": 0.3613, + "fro_norm": 18.0866 + }, + { + "layer": 3, + "rank90": 235, + "rank90_frac": 0.459, + "fro_norm": 21.8594 + }, + { + "layer": 4, + "rank90": 249, + "rank90_frac": 0.4863, + "fro_norm": 26.9032 + }, + { + "layer": 5, + "rank90": 256, + "rank90_frac": 0.5, + "fro_norm": 20.4489 + } + ], + "mean_rank90": 223.83 + }, + "load_time": 12.2 + }, + "step16000": { + "revision": "step16000", + "h3": { + "recovery": [ + -0.0, + -0.0312, + 0.0265, + 0.271, + 0.3668, + 0.6868 + ], + "L_crit_90": null, + "L_crit_99": null, + "mu_baseline": -13.3378, + "mu_d10": -14.4215, + "denom": -1.0837, + "n_prompts": 50 + }, + "wo_rank": { + "layers": [ + { + "layer": 0, + "rank90": 189, + "rank90_frac": 0.3691, + "fro_norm": 16.3548 + }, + { + "layer": 1, + "rank90": 228, + "rank90_frac": 0.4453, + "fro_norm": 18.3408 + }, + { + "layer": 2, + "rank90": 182, + "rank90_frac": 0.3555, + "fro_norm": 20.2797 + }, + { + "layer": 3, + "rank90": 232, + "rank90_frac": 0.4531, + "fro_norm": 22.8214 + }, + { + "layer": 4, + "rank90": 255, + "rank90_frac": 0.498, + "fro_norm": 30.659 + }, + { + "layer": 5, + "rank90": 262, + "rank90_frac": 0.5117, + "fro_norm": 23.3895 + } + ], + "mean_rank90": 224.67 + }, + "load_time": 10.5 + }, + "step32000": { + "revision": "step32000", + "h3": { + "recovery": [ + -0.0, + -0.2954, + 0.0591, + 0.374, + 0.2268, + 0.8518 + ], + "L_crit_90": null, + "L_crit_99": null, + "mu_baseline": -14.1658, + "mu_d10": -14.8156, + "denom": -0.6498, + "n_prompts": 50 + }, + "wo_rank": { + "layers": [ + { + "layer": 0, + "rank90": 191, + "rank90_frac": 0.373, + "fro_norm": 16.7226 + }, + { + "layer": 1, + "rank90": 230, + "rank90_frac": 0.4492, + "fro_norm": 18.5145 + }, + { + "layer": 2, + "rank90": 180, + "rank90_frac": 0.3516, + "fro_norm": 20.6984 + }, + { + "layer": 3, + "rank90": 227, + "rank90_frac": 0.4434, + "fro_norm": 22.4047 + }, + { + "layer": 4, + "rank90": 256, + "rank90_frac": 0.5, + "fro_norm": 31.7069 + }, + { + "layer": 5, + "rank90": 264, + "rank90_frac": 0.5156, + "fro_norm": 18.9558 + } + ], + "mean_rank90": 224.67 + }, + "load_time": 9.3 + }, + "step143000": { + "revision": "step143000", + "h3": { + "recovery": [ + 0.0, + -1.6746, + -0.7017, + -0.6383, + 1.2949, + 0.9589 + ], + "L_crit_90": 4, + "L_crit_99": 4, + "mu_baseline": -16.482, + "mu_d10": -15.7736, + "denom": 0.7085, + "n_prompts": 50 + }, + "wo_rank": { + "layers": [ + { + "layer": 0, + "rank90": 193, + "rank90_frac": 0.377, + "fro_norm": 10.7524 + }, + { + "layer": 1, + "rank90": 230, + "rank90_frac": 0.4492, + "fro_norm": 11.6785 + }, + { + "layer": 2, + "rank90": 180, + "rank90_frac": 0.3516, + "fro_norm": 13.7552 + }, + { + "layer": 3, + "rank90": 188, + "rank90_frac": 0.3672, + "fro_norm": 18.607 + }, + { + "layer": 4, + "rank90": 205, + "rank90_frac": 0.4004, + "fro_norm": 8.1702 + }, + { + "layer": 5, + "rank90": 240, + "rank90_frac": 0.4688, + "fro_norm": 8.2678 + } + ], + "mean_rank90": 206.0 + }, + "load_time": 2.1 + } +} \ No newline at end of file diff --git a/data/exp_t2/icl_spatial_gpt2-medium.json b/data/exp_t2/icl_spatial_gpt2-medium.json new file mode 100644 index 0000000000000000000000000000000000000000..ebdc77cd1d3413db8050a26a9527020bc5cb9b1a --- /dev/null +++ b/data/exp_t2/icl_spatial_gpt2-medium.json @@ -0,0 +1,117 @@ +{ + "model": "gpt2-medium", + "n_layers": 24, + "L_crit_known": 23, + "L_jump_detected": 22, + "dist_easy": 50, + "dist_hard": 500, + "n_prompts": 50, + "icl_score": { + "0": 0.01107959747314453, + "1": 0.9006196594238282, + "2": 1.6509499740600586, + "3": 1.849005813598633, + "4": 0.75073974609375, + "5": 0.1664865493774414, + "6": 1.3327793884277344, + "7": 2.555304012298584, + "8": 4.202056732177734, + "9": 4.950149993896485, + "10": 4.938382263183594, + "11": 7.841646461486817, + "12": 7.6333463096618654, + "13": 9.802567367553712, + "14": 14.741650924682617, + "15": 17.086366357803346, + "16": 17.862788043022157, + "17": 23.91103755950928, + "18": 25.81832295894623, + "19": 32.52337613683194, + "20": 38.87951512813569, + "21": 44.101259082196194, + "22": 52.011237938553094, + "23": 58.23494463443756, + "24": 2.0833090686798097 + }, + "mean_delta": { + "0": -0.01107959747314453, + "1": -0.9006196594238282, + "2": -1.6509499740600586, + "3": -1.849005813598633, + "4": -0.75073974609375, + "5": -0.1664865493774414, + "6": -1.3327793884277344, + "7": -2.555304012298584, + "8": -4.202056732177734, + "9": -4.950149993896485, + "10": -4.938382263183594, + "11": -7.841646461486817, + "12": -7.6333463096618654, + "13": -9.802567367553712, + "14": -14.741650924682617, + "15": -17.086366357803346, + "16": -17.862788043022157, + "17": -23.91103755950928, + "18": -25.81832295894623, + "19": -32.52337613683194, + "20": -38.87951512813569, + "21": -44.101259082196194, + "22": -52.011237938553094, + "23": -58.23494463443756, + "24": -2.0833090686798097 + }, + "mean_easy": { + "0": -15.701723613739013, + "1": -60.01761688232422, + "2": -60.68398200988769, + "3": -60.7508895111084, + "4": -63.70591766357422, + "5": -66.5406484222412, + "6": -67.88113548278808, + "7": -69.42489345550537, + "8": -71.10670009613037, + "9": -76.11728954315186, + "10": -75.0453564453125, + "11": -75.19998359680176, + "12": -76.44891380310058, + "13": -77.41143787384033, + "14": -79.31632915496826, + "15": -85.85235014915466, + "16": -91.33954855442047, + "17": -94.16897157669068, + "18": -96.06297297000884, + "19": -94.72350374598057, + "20": -96.96555620670318, + "21": -92.7302533499022, + "22": -86.60161088943481, + "23": -83.87639896392822, + "24": -7.816802892684937 + }, + "mean_hard": { + "0": -15.712803211212158, + "1": -60.918236541748044, + "2": -62.334931983947754, + "3": -62.599895324707035, + "4": -64.45665740966797, + "5": -66.70713497161866, + "6": -69.21391487121582, + "7": -71.98019746780396, + "8": -75.3087568283081, + "9": -81.06743953704834, + "10": -79.98373870849609, + "11": -83.04163005828858, + "12": -84.08226011276246, + "13": -87.21400524139405, + "14": -94.05798007965087, + "15": -102.938716506958, + "16": -109.20233659744262, + "17": -118.08000913619995, + "18": -121.88129592895508, + "19": -127.2468798828125, + "20": -135.84507133483888, + "21": -136.83151243209838, + "22": -138.6128488279879, + "23": -142.11134359836578, + "24": -9.900111961364747 + } +} \ No newline at end of file diff --git a/data/exp_t2/icl_spatial_pythia-70m.json b/data/exp_t2/icl_spatial_pythia-70m.json new file mode 100644 index 0000000000000000000000000000000000000000..ca6d3813cf8dbdb6b54cc3eccc7e3dfba6afc5a8 --- /dev/null +++ b/data/exp_t2/icl_spatial_pythia-70m.json @@ -0,0 +1,45 @@ +{ + "model": "EleutherAI/pythia-70m", + "n_layers": 6, + "L_crit_known": 4, + "L_jump_detected": 5, + "dist_easy": 50, + "dist_hard": 500, + "n_prompts": 50, + "icl_score": { + "0": -0.0, + "1": -0.4177438926696777, + "2": -1.0909214782714844, + "3": -10.957723121643067, + "4": -7.775770399570465, + "5": -2.8682076358795165, + "6": -0.24704144954681395 + }, + "mean_delta": { + "0": 0.0, + "1": 0.4177438926696777, + "2": 1.0909214782714844, + "3": 10.957723121643067, + "4": 7.775770399570465, + "5": 2.8682076358795165, + "6": 0.24704144954681395 + }, + "mean_easy": { + "0": -11.925345001220704, + "1": -16.562915420532228, + "2": -21.35541399002075, + "3": -47.06592821121216, + "4": -46.99601957321167, + "5": -35.63927864074707, + "6": -15.971736564636231 + }, + "mean_hard": { + "0": -11.925345001220704, + "1": -16.145171527862548, + "2": -20.264492511749268, + "3": -36.10820508956909, + "4": -39.22024917364121, + "5": -32.77107100486755, + "6": -15.724695115089416 + } +} \ No newline at end of file diff --git a/data/exp_w2/rank_profile.json b/data/exp_w2/rank_profile.json new file mode 100644 index 0000000000000000000000000000000000000000..d3e5f16e8e12d689c5fae095724cc0a9cd1d579e --- /dev/null +++ b/data/exp_w2/rank_profile.json @@ -0,0 +1,489 @@ +{ + "pythia-70m": { + "model": "pythia-70m", + "pe": "RoPE", + "d_model": 512, + "L_crit": 4, + "f_active": 0.374, + "rank90_pred": 191, + "k_thresh_rho": 191, + "layers": [ + { + "layer": 0, + "label": "pre_Lcrit", + "rank90": 193, + "rank50": 55, + "rho": 0.8981, + "fro_norm": 10.7524, + "rank90_frac": 0.377, + "rank90_err_pct": 1.0 + }, + { + "layer": 1, + "label": "pre_Lcrit", + "rank90": 230, + "rank50": 78, + "rho": 0.8394, + "fro_norm": 11.6785, + "rank90_frac": 0.4492, + "rank90_err_pct": 20.4 + }, + { + "layer": 2, + "label": "pre_Lcrit", + "rank90": 180, + "rank50": 53, + "rho": 0.9157, + "fro_norm": 13.7552, + "rank90_frac": 0.3516, + "rank90_err_pct": -5.8 + }, + { + "layer": 3, + "label": "pre_Lcrit", + "rank90": 188, + "rank50": 53, + "rho": 0.9051, + "fro_norm": 18.607, + "rank90_frac": 0.3672, + "rank90_err_pct": -1.6 + }, + { + "layer": 4, + "label": "at_Lcrit", + "rank90": 205, + "rank50": 69, + "rho": 0.8797, + "fro_norm": 8.1702, + "rank90_frac": 0.4004, + "rank90_err_pct": 7.3 + }, + { + "layer": 5, + "label": "post_Lcrit", + "rank90": 240, + "rank50": 82, + "rho": 0.8183, + "fro_norm": 8.2678, + "rank90_frac": 0.4688, + "rank90_err_pct": 25.7 + } + ], + "summary": { + "mean_rank90": 206.0, + "std_rank90": 21.99, + "pre_Lcrit_mean_rho": 0.8896, + "post_Lcrit_mean_rho": 0.8183 + } + }, + "gpt2-medium": { + "model": "gpt2-medium", + "pe": "AbsPE", + "d_model": 1024, + "L_crit": 23, + "f_active": null, + "rank90_pred": null, + "k_thresh_rho": 512, + "layers": [ + { + "layer": 0, + "label": "pre_Lcrit", + "rank90": 419, + "rank50": 102, + "rho": 0.9437, + "fro_norm": 43.0384, + "rank90_frac": 0.4092 + }, + { + "layer": 1, + "label": "pre_Lcrit", + "rank90": 268, + "rank50": 20, + "rho": 0.9792, + "fro_norm": 92.178, + "rank90_frac": 0.2617 + }, + { + "layer": 2, + "label": "pre_Lcrit", + "rank90": 392, + "rank50": 97, + "rho": 0.9542, + "fro_norm": 72.4524, + "rank90_frac": 0.3828 + }, + { + "layer": 3, + "label": "pre_Lcrit", + "rank90": 429, + "rank50": 126, + "rho": 0.9423, + "fro_norm": 69.9871, + "rank90_frac": 0.4189 + }, + { + "layer": 4, + "label": "pre_Lcrit", + "rank90": 431, + "rank50": 125, + "rho": 0.9418, + "fro_norm": 72.3096, + "rank90_frac": 0.4209 + }, + { + "layer": 5, + "label": "pre_Lcrit", + "rank90": 437, + "rank50": 131, + "rho": 0.9398, + "fro_norm": 78.4996, + "rank90_frac": 0.4268 + }, + { + "layer": 6, + "label": "pre_Lcrit", + "rank90": 431, + "rank50": 121, + "rho": 0.9413, + "fro_norm": 84.0501, + "rank90_frac": 0.4209 + }, + { + "layer": 7, + "label": "pre_Lcrit", + "rank90": 430, + "rank50": 130, + "rho": 0.9424, + "fro_norm": 87.7387, + "rank90_frac": 0.4199 + }, + { + "layer": 8, + "label": "pre_Lcrit", + "rank90": 415, + "rank50": 119, + "rho": 0.9479, + "fro_norm": 91.8613, + "rank90_frac": 0.4053 + }, + { + "layer": 9, + "label": "pre_Lcrit", + "rank90": 406, + "rank50": 96, + "rho": 0.9503, + "fro_norm": 98.2563, + "rank90_frac": 0.3965 + }, + { + "layer": 10, + "label": "pre_Lcrit", + "rank90": 393, + "rank50": 89, + "rho": 0.9538, + "fro_norm": 97.6642, + "rank90_frac": 0.3838 + }, + { + "layer": 11, + "label": "pre_Lcrit", + "rank90": 420, + "rank50": 113, + "rho": 0.9453, + "fro_norm": 97.9002, + "rank90_frac": 0.4102 + }, + { + "layer": 12, + "label": "pre_Lcrit", + "rank90": 405, + "rank50": 114, + "rho": 0.9504, + "fro_norm": 96.6644, + "rank90_frac": 0.3955 + }, + { + "layer": 13, + "label": "pre_Lcrit", + "rank90": 428, + "rank50": 130, + "rho": 0.9426, + "fro_norm": 94.9417, + "rank90_frac": 0.418 + }, + { + "layer": 14, + "label": "pre_Lcrit", + "rank90": 460, + "rank50": 145, + "rho": 0.9289, + "fro_norm": 95.5705, + "rank90_frac": 0.4492 + }, + { + "layer": 15, + "label": "pre_Lcrit", + "rank90": 419, + "rank50": 117, + "rho": 0.9465, + "fro_norm": 100.0642, + "rank90_frac": 0.4092 + }, + { + "layer": 16, + "label": "pre_Lcrit", + "rank90": 458, + "rank50": 144, + "rho": 0.9291, + "fro_norm": 103.4509, + "rank90_frac": 0.4473 + }, + { + "layer": 17, + "label": "pre_Lcrit", + "rank90": 448, + "rank50": 140, + "rho": 0.9345, + "fro_norm": 105.379, + "rank90_frac": 0.4375 + }, + { + "layer": 18, + "label": "pre_Lcrit", + "rank90": 479, + "rank50": 157, + "rho": 0.9192, + "fro_norm": 108.1156, + "rank90_frac": 0.4678 + }, + { + "layer": 19, + "label": "pre_Lcrit", + "rank90": 505, + "rank50": 162, + "rho": 0.9046, + "fro_norm": 115.4507, + "rank90_frac": 0.4932 + }, + { + "layer": 20, + "label": "pre_Lcrit", + "rank90": 498, + "rank50": 163, + "rho": 0.9089, + "fro_norm": 117.7071, + "rank90_frac": 0.4863 + }, + { + "layer": 21, + "label": "pre_Lcrit", + "rank90": 499, + "rank50": 153, + "rho": 0.908, + "fro_norm": 123.4158, + "rank90_frac": 0.4873 + }, + { + "layer": 22, + "label": "pre_Lcrit", + "rank90": 490, + "rank50": 141, + "rho": 0.913, + "fro_norm": 132.9368, + "rank90_frac": 0.4785 + }, + { + "layer": 23, + "label": "at_Lcrit", + "rank90": 408, + "rank50": 45, + "rho": 0.9462, + "fro_norm": 145.1893, + "rank90_frac": 0.3984 + } + ], + "summary": { + "mean_rank90": 432.0, + "std_rank90": 47.56, + "pre_Lcrit_mean_rho": 0.9377, + "post_Lcrit_mean_rho": 0.0 + } + }, + "pythia-1b": { + "model": "pythia-1b", + "pe": "RoPE", + "d_model": 2048, + "L_crit": 15, + "f_active": 0.128, + "rank90_pred": 262, + "k_thresh_rho": 262, + "layers": [ + { + "layer": 0, + "label": "pre_Lcrit", + "rank90": 810, + "rank50": 211, + "rho": 0.563, + "fro_norm": 36.8091, + "rank90_frac": 0.3955, + "rank90_err_pct": 209.2 + }, + { + "layer": 1, + "label": "pre_Lcrit", + "rank90": 894, + "rank50": 265, + "rho": 0.4974, + "fro_norm": 32.1104, + "rank90_frac": 0.4365, + "rank90_err_pct": 241.2 + }, + { + "layer": 2, + "label": "pre_Lcrit", + "rank90": 930, + "rank50": 280, + "rho": 0.4807, + "fro_norm": 35.3559, + "rank90_frac": 0.4541, + "rank90_err_pct": 255.0 + }, + { + "layer": 3, + "label": "pre_Lcrit", + "rank90": 893, + "rank50": 278, + "rho": 0.4817, + "fro_norm": 36.2661, + "rank90_frac": 0.436, + "rank90_err_pct": 240.8 + }, + { + "layer": 4, + "label": "pre_Lcrit", + "rank90": 917, + "rank50": 274, + "rho": 0.4865, + "fro_norm": 37.3924, + "rank90_frac": 0.4478, + "rank90_err_pct": 250.0 + }, + { + "layer": 5, + "label": "pre_Lcrit", + "rank90": 899, + "rank50": 273, + "rho": 0.4879, + "fro_norm": 40.6263, + "rank90_frac": 0.439, + "rank90_err_pct": 243.1 + }, + { + "layer": 6, + "label": "pre_Lcrit", + "rank90": 865, + "rank50": 247, + "rho": 0.518, + "fro_norm": 42.9162, + "rank90_frac": 0.4224, + "rank90_err_pct": 230.2 + }, + { + "layer": 7, + "label": "pre_Lcrit", + "rank90": 907, + "rank50": 295, + "rho": 0.4617, + "fro_norm": 42.1987, + "rank90_frac": 0.4429, + "rank90_err_pct": 246.2 + }, + { + "layer": 8, + "label": "pre_Lcrit", + "rank90": 837, + "rank50": 235, + "rho": 0.5343, + "fro_norm": 44.9326, + "rank90_frac": 0.4087, + "rank90_err_pct": 219.5 + }, + { + "layer": 9, + "label": "pre_Lcrit", + "rank90": 903, + "rank50": 291, + "rho": 0.4654, + "fro_norm": 45.1014, + "rank90_frac": 0.4409, + "rank90_err_pct": 244.7 + }, + { + "layer": 10, + "label": "pre_Lcrit", + "rank90": 960, + "rank50": 322, + "rho": 0.4331, + "fro_norm": 45.0008, + "rank90_frac": 0.4688, + "rank90_err_pct": 266.4 + }, + { + "layer": 11, + "label": "pre_Lcrit", + "rank90": 901, + "rank50": 290, + "rho": 0.4662, + "fro_norm": 52.5301, + "rank90_frac": 0.4399, + "rank90_err_pct": 243.9 + }, + { + "layer": 12, + "label": "pre_Lcrit", + "rank90": 947, + "rank50": 311, + "rho": 0.4436, + "fro_norm": 52.3493, + "rank90_frac": 0.4624, + "rank90_err_pct": 261.5 + }, + { + "layer": 13, + "label": "pre_Lcrit", + "rank90": 1001, + "rank50": 348, + "rho": 0.4063, + "fro_norm": 54.4121, + "rank90_frac": 0.4888, + "rank90_err_pct": 282.1 + }, + { + "layer": 14, + "label": "pre_Lcrit", + "rank90": 1015, + "rank50": 352, + "rho": 0.4052, + "fro_norm": 51.9307, + "rank90_frac": 0.4956, + "rank90_err_pct": 287.4 + }, + { + "layer": 15, + "label": "at_Lcrit", + "rank90": 1039, + "rank50": 364, + "rho": 0.3934, + "fro_norm": 51.225, + "rank90_frac": 0.5073, + "rank90_err_pct": 296.6 + } + ], + "summary": { + "mean_rank90": 919.9, + "std_rank90": 59.62, + "pre_Lcrit_mean_rho": 0.4754, + "post_Lcrit_mean_rho": 0.0 + } + } +} \ No newline at end of file diff --git a/data/exp_wqk_spectral/SmolLM2-1.7B_wqk_spectral.json b/data/exp_wqk_spectral/SmolLM2-1.7B_wqk_spectral.json new file mode 100644 index 0000000000000000000000000000000000000000..7ba22031ba1f7a3203840180a0bd67d6ddad9668 --- /dev/null +++ b/data/exp_wqk_spectral/SmolLM2-1.7B_wqk_spectral.json @@ -0,0 +1,2204 @@ +{ + "model": "HuggingFaceTB/SmolLM2-1.7B", + "short_name": "SmolLM2-1.7B", + "theta": 130000, + "T_train": 8192, + "d_head": 64, + "n_pairs": 32, + "k_dead": 22, + "n_heads": 32, + "n_kv": 32, + "n_layers": 24, + "global_mean_snr": 0.7871008142346233, + "snr_real_inv": 1.2704852820821941, + "global_min_snr": 0.6920461542534845, + "global_max_snr": 0.8679044814409683, + "layers": [ + { + "layer": 0, + "mean_snr": 0.8472789805637493, + "per_head_snr": [ + 0.672663721348349, + 0.6883447212741385, + 0.8773481886062032, + 0.7943805213551244, + 1.2406555957121996, + 1.0824009891112767, + 1.1408103497891202, + 0.7843810601262176, + 0.7648286210120021, + 1.1633530433674002, + 0.8735163104873004, + 0.588978830995693, + 0.6705013113506334, + 1.0509324833711837, + 0.8714729426613103, + 0.6623366019301178, + 1.004479575260129, + 0.6837502209693415, + 0.8748574213005942, + 0.6644200640178151, + 1.0205621607144586, + 0.9616347210167229, + 0.7781940091071174, + 0.513731135844974, + 0.7122781556697827, + 0.5997793533445157, + 1.0356549949778613, + 0.8993289973575164, + 0.8154874237579457, + 0.8732401044509261, + 1.0214034708080375, + 0.7272202769439685 + ], + "mean_band_energy": [ + 157.45273021139005, + 99.73918036998927, + 101.54826518381678, + 91.38830024169833, + 99.18922049764899, + 104.88888743134471, + 129.9092394372547, + 148.10009115527413, + 147.786823012052, + 168.3304230891212, + 189.76765092149606, + 190.33731571919319, + 147.1983109587169, + 174.85411956160112, + 171.68304234419324, + 181.56694252770163, + 145.61346753126887, + 101.72575211989766, + 102.05459827057331, + 94.23630494927878, + 101.08828853389713, + 103.74876788874163, + 130.69010076665134, + 149.72910070908375, + 155.6225122411722, + 157.85654595491474, + 203.46476075298574, + 181.45964430074724, + 161.80895362145682, + 174.67782031041298, + 172.84692530159992, + 171.8200751223551 + ], + "dft_magnitude": [ + 4612.1841610375295, + 17.446683944352017, + 691.2316281705235, + 5.115795650026047, + 171.46150202448388, + 8.875542933968582, + 154.78470365809108, + 27.777920957134967, + 150.0500430064778, + 13.269105056578447, + 73.21865740020556, + 30.430661849363425, + 95.33180242114717, + 14.5027705007447, + 69.26721535759293, + 69.0717013829497, + 23.265618134818396 + ] + }, + { + "layer": 1, + "mean_snr": 0.8335919550092413, + "per_head_snr": [ + 0.9081535231557255, + 0.8026214199633956, + 0.5767513505226287, + 0.8630363509588781, + 0.8745887683452239, + 0.8753943266110317, + 0.8454335719085515, + 0.9205088970682073, + 0.7716151939757165, + 0.8279817370539085, + 0.8472338239046331, + 0.7182777538809236, + 0.9213197482778954, + 0.9005223635573618, + 0.8242365153432621, + 0.9038970777823828, + 0.9024842886431825, + 0.8303856890415026, + 0.8483698118267698, + 0.8609346316223069, + 0.7856846463234932, + 0.7874834206879897, + 0.8086262826462342, + 0.7516283789524775, + 0.9566420000178179, + 0.706790420041472, + 0.7961304600949407, + 0.8341206823145304, + 0.9248724130585768, + 0.874252088705754, + 0.7937300038312448, + 0.8312349201777028 + ], + "mean_band_energy": [ + 63.513351772114895, + 60.31566480901624, + 68.48935127779573, + 71.95574809684524, + 75.28327724833869, + 82.83961130616939, + 83.47388913889492, + 91.9232768050351, + 94.76976388076078, + 98.2646842540583, + 118.12722460362788, + 90.23407815009355, + 99.49274560123946, + 114.05197610407912, + 116.3393778198811, + 115.58132771164608, + 64.70492027337646, + 60.99460648940894, + 69.03049020075855, + 67.63777909046917, + 78.66666496440604, + 81.25439652157561, + 83.01366890809045, + 92.47975591726929, + 96.46579366066314, + 99.97050331783194, + 116.83364424928936, + 93.22471382953371, + 95.37854555094452, + 117.34397053560795, + 116.64367714392259, + 116.29071751982968 + ], + "dft_magnitude": [ + 2894.589196752574, + 3.633921681906169, + 324.006409150626, + 3.1622009022508193, + 147.6088703262217, + 4.9928113300426515, + 164.94402558682148, + 7.500350522155032, + 106.4841182742936, + 6.6902452561146415, + 8.617254714102971, + 9.46303870290196, + 101.53579781670885, + 11.426367431031823, + 66.97363227831183, + 16.46963969165696, + 14.136424164364826 + ] + }, + { + "layer": 2, + "mean_snr": 0.829875622998896, + "per_head_snr": [ + 0.8399524324581211, + 0.9409789765719437, + 0.8505319824579196, + 0.8518286631783821, + 0.8726443671378125, + 0.7144801230948135, + 0.7124848086410278, + 0.7725873238488732, + 0.8300901075810173, + 0.8128183468787378, + 0.8858432701216533, + 0.8738264240324563, + 0.9069895550122964, + 0.8747638623740167, + 0.9129799450240602, + 0.8814479461294868, + 0.85454813665926, + 0.9352539735090889, + 0.7280108184629228, + 0.9436377283613295, + 0.8834537016778534, + 0.8800429676345418, + 0.851823567357151, + 0.6436828608010494, + 0.7997858928751996, + 0.7952030867260153, + 0.8066606479281363, + 0.7800432849707979, + 0.7471063250190698, + 0.9046906382525257, + 0.6979612844005411, + 0.7698668867865698 + ], + "mean_band_energy": [ + 47.15704843674553, + 52.90619966645231, + 61.7818133459855, + 68.31534817300178, + 69.74614592503355, + 75.28656620178141, + 77.9185592760254, + 88.22134091495978, + 90.11401486023723, + 89.31970854209375, + 98.27014897920114, + 75.69215659606554, + 86.48868538104716, + 98.15305508256648, + 101.94884312354026, + 103.66805988721143, + 51.36370921861407, + 50.31790649400501, + 64.16983540100219, + 68.35913087485645, + 68.51773930498875, + 74.41799570761617, + 79.79213290008286, + 87.97709180300055, + 89.33356226017072, + 88.08991480186285, + 99.2421167662252, + 79.03558148966874, + 86.82246513754473, + 97.40852628158314, + 100.95139973235185, + 101.66311850393049 + ], + "dft_magnitude": [ + 2572.449921069452, + 4.636609174241639, + 252.54990161028843, + 8.799012455040742, + 147.4043086336752, + 8.07701708884962, + 158.46721583419128, + 5.867037087234179, + 105.58506218575764, + 8.265771611646361, + 30.433418498184533, + 9.37785178416052, + 88.83234138310307, + 3.3937398784493342, + 71.17854928008988, + 8.972212904080815, + 25.21348097185978 + ] + }, + { + "layer": 3, + "mean_snr": 0.8289647833450933, + "per_head_snr": [ + 0.8654108371596502, + 0.8785977955971896, + 0.8562638863711277, + 0.6646064442718329, + 0.7984471014970038, + 0.8112176148120369, + 0.9087573607319279, + 0.9370678947563855, + 0.8778321132946596, + 0.8244285462719675, + 0.5949997702786975, + 0.8915433781685912, + 0.7297557332692132, + 0.8568230642301273, + 0.7676845117723904, + 0.9024882850214375, + 0.724987741332448, + 0.9002495171887016, + 0.7392578548529356, + 0.66904352188815, + 0.900149957188572, + 0.9660851767103269, + 0.8618333802389264, + 0.8617283312539548, + 0.8478110934893466, + 0.8329896553293449, + 0.8136509583284418, + 0.8977226582720431, + 0.6924564459600523, + 0.8599152654907554, + 0.9777588175708541, + 0.8153083544438923 + ], + "mean_band_energy": [ + 49.43028261447878, + 52.17693698132376, + 57.87284556619322, + 62.195405031017074, + 61.81246145420415, + 71.42602671337234, + 74.91869970011709, + 83.30392287964071, + 82.52327668034272, + 86.17198411583799, + 90.5966176146545, + 77.37123913287755, + 79.45687141123621, + 89.53857450224785, + 92.75698355377082, + 92.46347902559451, + 50.4479490663819, + 53.655346817381215, + 55.195027741925365, + 62.65660546370237, + 65.12321517543126, + 66.80335416141213, + 74.01253683949936, + 82.12005381697224, + 83.20773005040756, + 87.09827258122095, + 91.64343553972063, + 77.24145139563151, + 81.41819572980198, + 90.6977535989753, + 93.24635543192764, + 92.14930263120058 + ], + "dft_magnitude": [ + 2410.732193018501, + 3.488607363376971, + 244.9039869721431, + 8.246485814838131, + 117.36266863960353, + 1.6294525939837101, + 118.78220985097315, + 9.214197955129508, + 83.19512872817936, + 6.972757141267439, + 17.56335772826062, + 10.929480471408597, + 58.413259961587435, + 6.573278488833911, + 49.6008723170947, + 6.781693444668732, + 43.40722467831483 + ] + }, + { + "layer": 4, + "mean_snr": 0.7969737187986556, + "per_head_snr": [ + 0.6937141315898829, + 0.7493406012361992, + 0.7476423043980321, + 0.7959579828909689, + 0.7579287237045766, + 0.8742129056487026, + 0.8375833021591133, + 0.8907078922996498, + 0.9440769500466774, + 0.8488665803787492, + 0.8836644812340011, + 0.6478453939945582, + 0.8225070140276453, + 0.7148205180930606, + 0.8737580558676022, + 0.8175970824352295, + 0.7980778839919028, + 0.6213896733736608, + 0.855710316812751, + 0.872841069023717, + 0.901053915408415, + 0.7937853693307123, + 0.8136951292884745, + 0.8409429408351772, + 0.6372129202288005, + 0.809574430909737, + 0.7372191108179097, + 0.9089867204854891, + 0.7935447322765816, + 0.8172654779619081, + 0.8579204821745465, + 0.5437149086325503 + ], + "mean_band_energy": [ + 41.44970649812293, + 42.970187001703884, + 49.586883506246934, + 55.736473582074126, + 54.88659508579988, + 61.32641150260028, + 66.11484817376541, + 73.07266034625044, + 77.11417192038051, + 83.9006051438003, + 89.57217328196734, + 83.36844357709165, + 84.96323269430407, + 97.37305263722885, + 96.0845909349867, + 97.4396945281128, + 41.36528673422554, + 43.806258590341436, + 48.058170503899206, + 51.451221440020795, + 58.782349100038914, + 60.24621377087009, + 66.55419851905083, + 73.40246033924721, + 77.94951520601616, + 84.36043468636133, + 88.8571073035737, + 81.62117613752173, + 84.77298995875881, + 96.25540537476857, + 94.47698273774367, + 93.6733634886286 + ], + "dft_magnitude": [ + 2300.592864305502, + 6.376831449760947, + 342.9477675879905, + 9.143621868611074, + 142.23648503091184, + 6.134094983128338, + 137.0613562001154, + 7.54106090210761, + 87.46239783683579, + 4.917323071218672, + 38.023750284367985, + 8.75132687464241, + 64.48266518113911, + 10.620168941456894, + 61.48955827098807, + 7.233651133751973, + 59.415259987741365 + ] + }, + { + "layer": 5, + "mean_snr": 0.8402868918981082, + "per_head_snr": [ + 0.8726196148430598, + 0.6704860165062908, + 0.8861702605418392, + 0.8296771025368944, + 0.7833194879006895, + 0.830150509738179, + 0.7183201731567803, + 0.83908133923367, + 0.8778107893644042, + 0.8808356679052909, + 0.782608792290032, + 0.8114442398317413, + 0.7824927019803504, + 0.9017178960639244, + 0.9261312653883611, + 0.8042404452234087, + 0.8453448955236602, + 0.8410364416610757, + 0.8888662634012829, + 0.8504489657056203, + 0.9486993068419063, + 0.9074186248595293, + 0.8504780787165311, + 0.8488755005625405, + 0.8964276570467772, + 0.9544860907068584, + 0.7580514747437168, + 0.7196502600735529, + 0.8008646056548954, + 0.8615902605172848, + 0.9123017519343065, + 0.8075340602850094 + ], + "mean_band_energy": [ + 50.03847486051714, + 49.076322174905, + 56.003730131469084, + 60.25822937626803, + 61.91254560197295, + 67.02254754519785, + 74.77673022605876, + 79.73944353651645, + 81.55847907997793, + 84.89887291568255, + 91.20749574455847, + 71.57825415733812, + 78.52645116789107, + 93.74392411887055, + 93.68969504542524, + 91.60277531210241, + 49.291398147100516, + 50.8207592279573, + 55.32653926896021, + 61.46597445082776, + 65.68357028300952, + 67.6318441228503, + 76.00693363485709, + 78.79380135366944, + 80.10554275269166, + 84.95967594685902, + 91.28230345732263, + 73.02395103881997, + 78.80922644195198, + 91.66942770254153, + 90.68987677459513, + 91.46871687396163 + ], + "dft_magnitude": [ + 2372.6635124727272, + 8.431459845482616, + 240.2360477773455, + 1.9400595453256593, + 130.30064272146933, + 11.394452985185021, + 130.12102808998839, + 2.852080531155226, + 85.00858183897091, + 5.23784084063184, + 16.52665432940986, + 6.880148851040269, + 63.10764012062045, + 5.743595724712726, + 57.60090693384222, + 0.6691246669832516, + 22.845527236008593 + ] + }, + { + "layer": 6, + "mean_snr": 0.8222313616859722, + "per_head_snr": [ + 0.8715624565984363, + 0.8424815355398136, + 0.7887949293232372, + 0.7076224513088463, + 0.731820777228839, + 0.8284084680386107, + 0.8862685188640845, + 0.5823293299785725, + 0.8672337508043741, + 0.8772096136537961, + 0.8492331739477242, + 0.7459934682511363, + 0.7583366832731937, + 0.8076131405828498, + 0.915712310609856, + 0.7105415420307071, + 0.8786150381157648, + 0.8448668581636555, + 0.8402235901646248, + 0.8936839639419758, + 0.9058904115895129, + 0.8760187727098954, + 0.6197471503464984, + 0.8700590497147719, + 0.8322782000591161, + 0.7980807907892099, + 0.8050720550970656, + 0.8824898204549824, + 0.9655902548078532, + 0.8623649822660664, + 0.84235313899168, + 0.8229073467043637 + ], + "mean_band_energy": [ + 44.325111063089516, + 47.650954054992496, + 51.353732221465656, + 52.96950439434726, + 60.58920358308305, + 63.713014071667125, + 68.06168668511849, + 72.9719557344531, + 76.9561472270677, + 81.19774156550062, + 86.53423792208423, + 77.1497932406454, + 77.60810905858278, + 88.62756110603559, + 89.20990320011329, + 89.75993719435809, + 43.72551600987209, + 46.54042678062342, + 50.0248556733211, + 53.967959927324536, + 58.95361084610103, + 65.35859856309733, + 69.49125547442713, + 73.39743537902214, + 75.85022063837796, + 81.67392050383039, + 86.91820746806995, + 77.22462092520908, + 75.9901180287686, + 88.36833142335635, + 91.47652376100383, + 87.27840253130123 + ], + "dft_magnitude": [ + 2254.9185962563106, + 0.7102666624862444, + 269.86404765366757, + 4.461911852687499, + 124.12279037000266, + 3.0788595289659653, + 118.586748009022, + 3.8469496937276593, + 81.9666189556323, + 8.907047969838832, + 30.33788095379872, + 5.741695104859863, + 54.41767983761829, + 2.404303400463728, + 51.39376559516476, + 7.589522312099479, + 40.78171853521758 + ] + }, + { + "layer": 7, + "mean_snr": 0.8030032342959676, + "per_head_snr": [ + 0.8739368291574683, + 0.7319496388023502, + 0.5811999787604805, + 0.8716495568552862, + 0.9489424318725171, + 0.6836996204711219, + 0.8961981679401619, + 0.8661233375585302, + 0.6836507949630708, + 0.6772907602160944, + 0.8011855130347035, + 0.8894000700635649, + 0.8863541365504992, + 0.7342196798490709, + 0.7206376190928091, + 0.924910397884372, + 0.7542173750852791, + 0.8350907135281509, + 0.8513818546748546, + 0.8247547786005301, + 0.6474819868053378, + 0.9307311294444448, + 0.9498188922500338, + 0.820049551905372, + 0.8100192004174303, + 0.7629789466853863, + 0.6968096822920191, + 0.9637868435530743, + 0.9031789867687658, + 0.6737137282754087, + 0.7326447310258903, + 0.7680965630868851 + ], + "mean_band_energy": [ + 42.04887554926106, + 41.12982711094829, + 45.525936838058556, + 49.214518716753055, + 54.95347532254399, + 59.90503716125921, + 64.38392411769703, + 70.75695536506971, + 73.35454331162734, + 79.04829129845339, + 85.93223846091158, + 80.57148562260788, + 80.87538676858232, + 88.90838940870321, + 88.95059977258657, + 87.9277312744521, + 41.47154555590601, + 42.733453470928275, + 47.207998181866955, + 50.63154910004317, + 51.73748473801193, + 59.87636621894887, + 62.73598252428061, + 70.50156353155694, + 73.60687821990007, + 80.34210826162506, + 85.93926919040734, + 78.78971188736348, + 81.2621975370676, + 87.74065594404968, + 87.52084720117406, + 89.00306079051981 + ], + "dft_magnitude": [ + 2184.587888453165, + 3.871813135487017, + 320.6360277144531, + 3.8667829019804434, + 123.15034689841582, + 8.686206689899509, + 108.66211802923875, + 4.028552464015166, + 78.53385304418953, + 7.066039538351106, + 37.33469448829479, + 2.870336789441628, + 58.549419128710795, + 5.929192660130318, + 46.82116429696143, + 5.38483713937615, + 49.57352187339893 + ] + }, + { + "layer": 8, + "mean_snr": 0.7862197604062422, + "per_head_snr": [ + 0.6729446327893167, + 0.9109444539070526, + 0.8690749080840432, + 0.7575753194141889, + 0.8170928479910804, + 0.6874771072622371, + 0.5739903565795813, + 0.7362676967780678, + 0.7705466156221716, + 0.7127725853428534, + 0.7924809383336655, + 0.765895273138182, + 0.901593524853076, + 0.7146591359642506, + 0.616413441169252, + 0.8389882059590775, + 0.8618761650353302, + 0.7220713014793929, + 0.8023384499881494, + 0.6681017127566523, + 0.7096431903068077, + 0.9225778946480659, + 0.9197358117983344, + 0.8992249243746242, + 0.9893526126013168, + 0.9452509648204157, + 0.5568956426990143, + 0.810569094392916, + 0.6810991471563916, + 0.9055533184504087, + 0.816902384948627, + 0.8091226743552051 + ], + "mean_band_energy": [ + 35.77234422184691, + 37.708183680519625, + 38.76504187676284, + 42.7587680420994, + 43.59813098413279, + 47.78710584511289, + 54.46271920828316, + 58.86143669654677, + 64.10714259768493, + 71.41042373466475, + 76.07686038158711, + 73.49329494595969, + 73.87605610961808, + 77.63121451302742, + 76.98920651089962, + 73.74265946697717, + 36.57688819885707, + 36.36340766707599, + 40.34752578435172, + 43.87237639912302, + 44.67678244248566, + 49.31063946947891, + 51.408308882957826, + 60.51673367016074, + 62.879630896562105, + 70.75956425377548, + 75.9732771995672, + 72.21242097725745, + 74.23268632020446, + 77.87594122410061, + 80.11697555006717, + 78.90124750731893 + ], + "dft_magnitude": [ + 1903.0649952590675, + 5.074845771379815, + 312.24378455759205, + 12.691875679493464, + 92.35511836938379, + 8.049685238359096, + 92.63513874446645, + 2.740871977427266, + 68.36710216270772, + 6.539172724292113, + 33.69861396329479, + 0.466516761692781, + 45.103542145059194, + 13.088685822431712, + 37.54898388257143, + 4.679334581981169, + 43.3458409273303 + ] + }, + { + "layer": 9, + "mean_snr": 0.842103734349513, + "per_head_snr": [ + 0.7699706278606373, + 0.9504110998476335, + 0.7268495939784073, + 0.8159072191279101, + 0.8219355609591762, + 0.8956340242561802, + 0.8823791482448765, + 0.9754593850793798, + 0.8312886802813272, + 0.7535336282373647, + 0.6720320343065193, + 0.8422597134333283, + 0.9502568359796196, + 0.9024222079878776, + 0.8539123743181131, + 0.8076946249845849, + 0.8015984110458987, + 0.8294348743253483, + 0.8857364897564188, + 0.9279485797927461, + 0.9171441342108724, + 0.9388916211942917, + 0.8112754515466201, + 0.8123177662934834, + 0.8522329017009878, + 0.7827474159231645, + 0.876200572285712, + 0.8629865521654203, + 0.6849156357444972, + 0.8548204022694149, + 0.884505023209718, + 0.7726169088368854 + ], + "mean_band_energy": [ + 41.39986074849088, + 44.935687367425466, + 48.01222577928173, + 50.643376203993675, + 55.561454159343334, + 60.671689176269, + 64.63105462524831, + 67.71690012413623, + 73.40220557108061, + 77.32802878613496, + 78.12938164630671, + 73.8290032782059, + 76.44069827782084, + 82.59162270676163, + 82.55685615717431, + 82.82114716950431, + 42.47084030771577, + 44.60603028388099, + 47.72356272244836, + 52.264985051159954, + 54.34731301613964, + 58.41297347523127, + 63.28593955677293, + 68.66175095657587, + 72.7459765631763, + 76.5029750309338, + 79.30701301600548, + 70.39497756684466, + 75.31427324448116, + 81.97610763592218, + 74.53883670189205, + 79.40890219841536 + ], + "dft_magnitude": [ + 2102.6336491047737, + 16.719682471438443, + 250.91175936155554, + 12.656047559291828, + 99.58970114184758, + 4.863049174445592, + 92.72863390110217, + 10.912044432103635, + 50.12728860267205, + 7.66025336109165, + 34.55079242741627, + 10.781717568438257, + 51.54469298589176, + 9.007384273522295, + 52.06844554726501, + 1.5599275410063997, + 42.89866491801695 + ] + }, + { + "layer": 10, + "mean_snr": 0.7174388742524721, + "per_head_snr": [ + 0.9436637869047306, + 0.5710554926859356, + 0.781989851526298, + 0.5039977898615375, + 0.7684447612832743, + 0.775240571318069, + 0.914603876185101, + 0.8370118418802113, + 0.8456692027801346, + 0.8329669095716147, + 0.7641911261853391, + 0.6069283614150708, + 0.5979949840242336, + 0.6248463566453251, + 0.6374376231775577, + 0.8251984135252121, + 0.5191476004465507, + 0.5652306527978158, + 0.7837929848728407, + 0.5186012326029574, + 0.8791950164778727, + 0.6712842555035781, + 0.8717372994179269, + 0.5611099044739168, + 0.8769515328171426, + 0.8301612003756346, + 0.6209716303236866, + 0.7271130706987463, + 0.4866390237782755, + 0.8101910853762628, + 0.765048601503471, + 0.6396279356427849 + ], + "mean_band_energy": [ + 25.793164762146368, + 27.118935797781226, + 29.38386412143501, + 32.82688872409017, + 37.69595339311219, + 40.96210384401354, + 47.04130371181755, + 53.92480064269416, + 58.74728108052328, + 68.90926679217336, + 74.57492454828622, + 85.07727039687984, + 81.92831407509725, + 87.77577933718419, + 91.16996110571522, + 83.51034892387688, + 25.779584758119324, + 27.025126249204273, + 30.496192095453914, + 33.66531066116386, + 37.07613815885807, + 41.64523397514892, + 48.464733980311955, + 53.864305451458144, + 59.38786280769758, + 67.76455066546417, + 75.46089348432885, + 84.43379540003521, + 83.648357816215, + 87.88063679194894, + 89.5365642382011, + 84.76608989285863 + ], + "dft_magnitude": [ + 1857.3355376832944, + 3.23148101144971, + 460.4226632294686, + 1.6666512736405212, + 168.38849870950764, + 1.3154971892645086, + 109.68760931929869, + 5.166359674898585, + 98.76386079665639, + 3.78055953910361, + 68.16425241773015, + 2.3933236158702558, + 52.191425561012345, + 6.578292294733081, + 50.494077201062616, + 3.843881359088556, + 64.96534940865672 + ] + }, + { + "layer": 11, + "mean_snr": 0.7300723880277145, + "per_head_snr": [ + 0.7751306546892562, + 0.803214736879171, + 0.9356936522779189, + 0.7081528736947195, + 0.581747428820635, + 0.7581430465631946, + 0.7449135063594482, + 0.8098999175630245, + 0.8324668246952929, + 0.7617139467043962, + 0.7800144817300484, + 0.9621709090576546, + 0.7980940119654645, + 0.6501840227859723, + 0.6404764600036223, + 0.6993301093671711, + 0.752656680484551, + 0.6362170262778637, + 0.6899631038518209, + 0.5672213943369621, + 0.6157806620321726, + 0.7553764645620641, + 0.6646496541003886, + 0.7683222021940311, + 0.6802499888301533, + 0.7731674513190766, + 0.70904058904732, + 0.5489651271309124, + 0.8721580095539782, + 0.7854831521456955, + 0.7550589400210551, + 0.546659387841825 + ], + "mean_band_energy": [ + 29.435839711826535, + 30.410849588499858, + 34.14017143638641, + 37.02628877931403, + 43.980897348031, + 48.66061711022763, + 56.345466200647586, + 65.95218190623268, + 71.17934719831237, + 77.72404669180287, + 84.36769810203312, + 82.84696901701972, + 83.33635702052237, + 94.53405651850356, + 91.57263681611295, + 84.22743721597084, + 29.224845454528513, + 30.73862849220891, + 34.867749551107565, + 38.04270709936121, + 45.330989163459016, + 49.79619709933725, + 56.37811181008212, + 66.3373685628805, + 71.0233657881264, + 77.20985718386052, + 84.21337090504068, + 84.05401574614731, + 83.59659391617913, + 94.48927189368457, + 92.73315885755846, + 89.27993385316188 + ], + "dft_magnitude": [ + 2043.0570260381676, + 6.040799681032734, + 445.6838319888272, + 9.605865459591676, + 166.5281036781103, + 4.534956456080361, + 128.4808493302826, + 5.644334169620323, + 89.22868678898696, + 7.014908727551537, + 46.25981146577903, + 2.910151703751857, + 61.73871057956072, + 3.542406543771891, + 58.39919370803613, + 4.2549431161744655, + 59.60382747825929 + ] + }, + { + "layer": 12, + "mean_snr": 0.6920461542534845, + "per_head_snr": [ + 0.6981606425081536, + 0.7897774727437467, + 0.4949519905659652, + 0.6822467451034606, + 0.889262367872331, + 0.7589802429448368, + 0.6742657536060581, + 0.6496392504421725, + 0.7303822984112786, + 0.9050909950894286, + 0.8503100999144676, + 0.61222310229357, + 0.6974602775913188, + 0.5761755805217923, + 0.5712127709797509, + 0.967567489584976, + 0.7065349211293116, + 0.589903167527985, + 0.751110399373786, + 0.7505987814709119, + 0.6734905450702507, + 0.5928485670149778, + 0.6554324922514217, + 0.6149329801150593, + 0.6974792058091488, + 0.6444346595103649, + 0.6477799777476277, + 0.6517255369707002, + 0.5636226982454332, + 0.552071219236343, + 0.8619341991180719, + 0.6438705053468009 + ], + "mean_band_energy": [ + 21.066048359306937, + 21.386315554570903, + 25.182815663052097, + 28.339296259201635, + 33.05399531683844, + 37.85908267796765, + 43.32889220824752, + 49.274031293778705, + 55.64141929019882, + 64.34875255189601, + 72.08190412166059, + 79.95665393787537, + 84.82374606727426, + 91.45634244377428, + 87.23105834094854, + 78.33188478314818, + 20.28430036003752, + 21.51848498031761, + 24.651043812922296, + 27.94446417838847, + 33.00342092214039, + 38.31034923131213, + 42.24737688252684, + 50.865318766950104, + 56.36427016585381, + 64.6512708028419, + 72.55429390730541, + 79.30481161162606, + 83.66560415054984, + 88.745639879171, + 86.51779780687946, + 78.51596847668469 + ], + "dft_magnitude": [ + 1742.5066548052473, + 2.580811253246797, + 488.5025377222589, + 7.364554121100702, + 190.1012474187295, + 4.3735338312790635, + 111.6235878473741, + 1.9336051097975302, + 79.37515953648469, + 3.912481713933591, + 59.60272549012413, + 2.0711651303017726, + 56.230141453435316, + 2.990572702505508, + 50.59754528322149, + 3.3836941601847035, + 59.11068005376194 + ] + }, + { + "layer": 13, + "mean_snr": 0.7361780462048149, + "per_head_snr": [ + 0.6031797529505017, + 0.8146725297817127, + 0.6880992233794181, + 0.6788128781046402, + 0.7195376119165298, + 0.8314957256344018, + 0.7002586113092155, + 0.6257826774222619, + 0.8443653315774061, + 0.5887310366640295, + 0.5293067617290682, + 0.7076498752952826, + 0.9055143951018638, + 0.7654734355335767, + 0.8375684378723866, + 0.914567021414996, + 0.6863820635944621, + 0.6609973313402352, + 0.6413679894885075, + 0.7817610891039061, + 0.7966122365033874, + 0.6671712876091501, + 0.6094399904891027, + 0.8439136455286013, + 0.8969716839612512, + 0.5645081984986549, + 0.8051312081712169, + 0.8570432856568264, + 0.848346817175902, + 0.6014448695819787, + 0.8030536670673519, + 0.7385368090962507 + ], + "mean_band_energy": [ + 27.429717064984413, + 29.076398782763984, + 32.95277680625465, + 38.36360577930153, + 41.280329700603815, + 46.391225970201404, + 52.83728464075869, + 57.83473054262009, + 65.93493673355732, + 74.03546735277466, + 78.14561809072092, + 77.8518532677486, + 80.48545131660794, + 82.27423245722, + 78.61928583523155, + 74.17552280764106, + 27.264486676795332, + 30.09172521024422, + 33.43743075284783, + 38.60731247777136, + 41.39162434137928, + 46.09484616676832, + 53.34401827091004, + 59.06807700995166, + 66.42206141660088, + 73.04390127914729, + 79.21775144472274, + 77.11773133455934, + 82.65975581934924, + 81.13698601372597, + 80.29953803841427, + 79.47676244966586 + ], + "dft_magnitude": [ + 1886.3624458518443, + 6.974556020064502, + 398.61750898094203, + 6.305382995894833, + 123.277509752902, + 6.514721375303584, + 101.71781512617116, + 7.49352171305289, + 69.01120524877476, + 6.881737349789307, + 52.76410154949926, + 6.681817831355442, + 55.145116438234105, + 6.863110883620091, + 54.81879595772485, + 4.3116900300166945, + 42.91831195236648 + ] + }, + { + "layer": 14, + "mean_snr": 0.7613076133455207, + "per_head_snr": [ + 0.6972822268598319, + 0.7330566958403352, + 0.6985495078123206, + 0.6330175316676631, + 0.7045510697675919, + 0.7257700616804538, + 0.8536143858984556, + 0.7728246474021063, + 0.7799140973978094, + 0.6937349221124532, + 0.7313267650477936, + 0.7158075799493743, + 0.7820645186424473, + 0.8500306886515817, + 0.9049281966383924, + 0.6731127786329228, + 0.5965332880353357, + 0.8271634538988453, + 0.721415220450336, + 0.8366041033746476, + 0.8466579598577333, + 0.9132384026599292, + 0.7271430738215201, + 0.8896140239958676, + 0.7986692211058137, + 0.7873291858259694, + 0.7408729981822749, + 0.8006071605947084, + 0.6566234070242192, + 0.6411660935955518, + 0.8476851564584482, + 0.7809352041739266 + ], + "mean_band_energy": [ + 29.036154381706965, + 31.16412174323962, + 36.39541192174939, + 41.164232203506764, + 46.60673478809191, + 49.626097453113246, + 56.08532708020532, + 64.38431510213586, + 67.99462971299967, + 74.285835544541, + 76.89031392241581, + 77.20405362464743, + 78.03427996675474, + 82.23411126524985, + 77.01477160211869, + 72.64773298308224, + 28.797561120580653, + 31.112946118629267, + 36.612415793745, + 40.54799594546016, + 46.16171757559011, + 49.19523614596415, + 57.11310591631896, + 63.0442292104762, + 67.23336143928259, + 74.42919261586528, + 78.16811749344924, + 75.73242596072095, + 76.23556584135247, + 82.96804562243567, + 78.37427955883584, + 74.51715448432631 + ], + "dft_magnitude": [ + 1921.0114741385914, + 3.7047782160204745, + 359.3636139916261, + 3.2928251462336884, + 123.65425095078135, + 4.495367988160204, + 100.44560806657906, + 7.494365586879761, + 66.10432863820886, + 4.2993336480758, + 34.01513732731333, + 1.353761145362796, + 51.34520422140347, + 2.004673776228841, + 45.95031405481929, + 3.553148662608834, + 47.5039779081967 + ] + }, + { + "layer": 15, + "mean_snr": 0.6977575150317943, + "per_head_snr": [ + 0.7840909421436858, + 0.7338534237156947, + 0.6600586559681284, + 0.8084311214636078, + 0.8283311580905246, + 0.6589947832319516, + 0.6987694606411703, + 0.5485935692431696, + 0.5270767108198152, + 0.7562095869903869, + 0.6669594490076108, + 0.6419052109660476, + 0.7514369569395704, + 0.5067895730577339, + 0.7551704343535287, + 0.7301022087163828, + 0.5707929394082765, + 0.6369504928577965, + 0.8424465409129964, + 0.6619188852253058, + 0.6145226296405979, + 0.84402974268322, + 0.7051065488976072, + 0.7662814069453399, + 0.6622656037815553, + 0.7146259356382326, + 0.6807029089991466, + 0.7535274550890203, + 0.6610371686527384, + 0.655452149272432, + 0.7888326414311811, + 0.712974186232961 + ], + "mean_band_energy": [ + 18.449411418729074, + 19.756268985718844, + 22.872188136975332, + 25.90674719023991, + 29.550118225826917, + 32.71077951047077, + 36.2757345423917, + 41.912191737229605, + 47.12391864850853, + 54.08824256167653, + 61.86350302465794, + 69.24554019799135, + 73.46661879168184, + 72.59999290285563, + 71.8905165606611, + 64.81492729598048, + 18.16264921673624, + 20.299144414373707, + 22.92147359849443, + 25.958842566489174, + 29.74412133893008, + 32.83151367247692, + 37.26023893540975, + 42.81860871894249, + 46.934766777546685, + 54.16084943846124, + 62.968010272880285, + 69.99291941992865, + 72.1572422130062, + 71.67490754696932, + 70.68701681716036, + 63.789555624391575 + ], + "dft_magnitude": [ + 1484.8885603037925, + 4.362227546765099, + 397.7684607943307, + 4.264291763624444, + 146.19089339920095, + 2.5928064092635643, + 77.96013375836093, + 3.5654450325845697, + 69.00455666214184, + 2.2877545524146177, + 53.59592637514046, + 2.4547391220987542, + 48.456328912587644, + 2.140636332009894, + 39.10208761610518, + 0.6541341791995168, + 40.23350326459979 + ] + }, + { + "layer": 16, + "mean_snr": 0.7404550182673515, + "per_head_snr": [ + 0.8143875481670447, + 0.7053625152029168, + 0.7417332338380649, + 0.7893969496384126, + 0.8700282151524895, + 0.7064638301588053, + 0.7389506879000208, + 0.7219898324846555, + 0.869541063625686, + 0.8303684806447468, + 0.6533980827714766, + 0.657049501089701, + 0.6882588279521622, + 0.8044896204069168, + 0.7238979949510851, + 0.9142147066910066, + 0.6958875178587364, + 0.6353949675436182, + 0.6476896422643194, + 0.6823643882390844, + 0.7230791667816799, + 0.7632209799306783, + 0.7986955643547577, + 0.6035208818475376, + 0.6090222714283063, + 0.6579325800349828, + 0.8729746841507662, + 0.7934515358492001, + 0.8191638314224847, + 0.7504147462747838, + 0.6635269224685175, + 0.7486898134306004 + ], + "mean_band_energy": [ + 23.72550654469795, + 25.428479956404292, + 28.362632460744507, + 31.351436305972292, + 36.567490855391235, + 38.71742160948301, + 42.87830991397313, + 49.78451742695194, + 53.98050240389878, + 60.59601679927686, + 68.33624379065722, + 68.23981242484417, + 71.78824904285261, + 70.16351263283968, + 65.72634027131565, + 67.79363271549072, + 23.67012324183261, + 25.490071054444662, + 27.182997073828425, + 32.01447880520764, + 36.117987083164174, + 39.707918286753255, + 44.22129544692686, + 48.904565523804, + 52.91789399814255, + 61.38615386403632, + 68.00346957042316, + 68.45897748882253, + 71.36928433540592, + 73.52271682366433, + 64.48903808240098, + 64.52780935495272 + ], + "dft_magnitude": [ + 1605.424885188604, + 1.9422400419290557, + 348.9071785004522, + 1.9282399194606592, + 110.8459086148505, + 7.7959040250470135, + 72.6570914064013, + 5.501771096863125, + 53.16461347917646, + 9.110833041804321, + 37.15122478274252, + 1.7631236365638683, + 57.80809248994537, + 6.843209959855452, + 49.499096358585135, + 4.226052408512414, + 46.75015695729269 + ] + }, + { + "layer": 17, + "mean_snr": 0.7769721529338809, + "per_head_snr": [ + 0.9521307283545933, + 0.8022206618445447, + 0.8419994906548803, + 0.7317595542394877, + 0.8372533531143299, + 0.7020321889342453, + 0.8383093015798025, + 0.7782597769939787, + 0.6849287675973934, + 0.8927490669473582, + 0.7148380801913142, + 0.7941656651311226, + 0.5986004967860995, + 0.7375004981871744, + 0.7882044397402542, + 0.6839552112183356, + 0.7772211178028988, + 0.9067880664795598, + 0.656070448988107, + 0.5407173939267977, + 0.7350399267398746, + 0.6087307887499791, + 0.8989146814355422, + 0.8267251674860859, + 0.6441788943093661, + 0.8820891544912016, + 0.7479451691510155, + 0.855306179954158, + 0.8712060162174887, + 0.7776706114976715, + 0.8033449127014471, + 0.9522530824380792 + ], + "mean_band_energy": [ + 24.486096775302624, + 26.943198741259614, + 28.888403848005552, + 32.734450367384056, + 34.081186644316176, + 37.455870474922136, + 41.399295544371476, + 45.41095254977601, + 49.100858961075744, + 53.70517012297411, + 57.648950260263895, + 61.222650833678344, + 61.24603877828307, + 61.31296635952708, + 59.65914019646382, + 57.374796114328845, + 24.763153651119264, + 26.44849980872045, + 28.200700142283583, + 32.1116498281922, + 34.691192298928044, + 38.52734828322608, + 42.112986644317274, + 44.71311880329255, + 49.96597700332368, + 54.347851582380876, + 57.6818350682596, + 60.804280026997006, + 60.69853696553951, + 62.027424894895276, + 58.195138174935146, + 57.10346553050568 + ], + "dft_magnitude": [ + 1465.0631852788488, + 1.752680025399833, + 264.47475287816366, + 3.591919503180432, + 88.710480811947, + 2.7472653983907076, + 57.50829849902306, + 3.688121658064525, + 46.37587031248164, + 3.189949936424169, + 36.77576168657443, + 0.5956240601400609, + 30.003571182832786, + 2.9412919290672477, + 35.9019507072135, + 2.864074137544184, + 39.42420336527175 + ] + }, + { + "layer": 18, + "mean_snr": 0.7844900681951337, + "per_head_snr": [ + 0.7691161549349601, + 0.8790366649141471, + 1.0134326740696422, + 0.8042186038082867, + 0.9350344579304608, + 0.6968301444348606, + 0.8996031646994227, + 0.8898904597659582, + 0.8446998542032862, + 0.8424292975017722, + 0.9235163642728277, + 0.6884118991442936, + 0.5960230771182011, + 0.6590331396516859, + 0.7166200016296225, + 0.8118065511704979, + 0.8361837810385623, + 0.6230634600718693, + 0.9162554690709128, + 0.6814384989647086, + 0.8439828730378652, + 0.9064351349574834, + 0.743230645471387, + 0.5630006508291134, + 0.6508276997946418, + 0.8951305597127831, + 0.695643614742125, + 0.8117114163733296, + 0.8040782890507947, + 0.8393111192320123, + 0.7435670909247829, + 0.580119369721982 + ], + "mean_band_energy": [ + 22.794689710889507, + 23.63373738201087, + 25.89344082908892, + 27.758651327431956, + 29.74516155523945, + 31.883144201047724, + 34.12517703008523, + 37.497497750399546, + 39.64375063233916, + 44.212390520649, + 50.355327617604374, + 57.74415115088584, + 56.51089262483412, + 59.435390876473704, + 57.63229818366609, + 55.666641740775255, + 22.64185755522969, + 24.19575976455389, + 26.22469214427911, + 27.193560926833705, + 30.249211637329168, + 32.12644102129076, + 34.195821094326135, + 36.73941803397577, + 39.71990088070014, + 44.29703392572557, + 50.74223574097795, + 57.34970577454743, + 57.30410953524273, + 59.26753934178615, + 54.906762093327586, + 50.73315443376657 + ], + "dft_magnitude": [ + 1302.4195470373131, + 7.751096942542012, + 258.0528561248183, + 7.305394809043314, + 100.677192462723, + 7.377050165005038, + 45.651927309267165, + 5.1340876686498875, + 47.52254092907639, + 6.09911544160421, + 34.243161338283485, + 4.346511343836603, + 26.86386131496691, + 3.1628219215105506, + 30.830155187313636, + 3.8619393313882, + 37.0488893069944 + ] + }, + { + "layer": 19, + "mean_snr": 0.7780297456501126, + "per_head_snr": [ + 0.7478586359490423, + 0.6693950933357291, + 0.753244642641342, + 0.7692805889975909, + 0.8716215857658347, + 0.8878141554884762, + 0.7877072264023782, + 0.8609690265604735, + 0.7209417066639217, + 0.7889245092215225, + 0.7695929268370484, + 0.924512772577297, + 0.8150900276579306, + 0.7354684962094963, + 0.8801680790002282, + 0.8978429335478305, + 0.8319283182162013, + 0.7629527987160548, + 0.8137853769310809, + 0.7429179822552051, + 0.6948610530347479, + 0.7378271738802734, + 0.7718668348650383, + 0.5771914547545016, + 0.7144283084007665, + 0.872079119293959, + 0.7554222779504675, + 0.7348618132201746, + 0.7825020958691763, + 0.7207318579859273, + 0.9132548048962685, + 0.5899081836776207 + ], + "mean_band_energy": [ + 21.213290469036593, + 23.179002960324354, + 24.250807953549753, + 25.912936007233643, + 27.785344932059758, + 29.832908021537754, + 31.115107716584866, + 32.60842707779888, + 37.32387791456526, + 40.184666519500375, + 42.8612998928227, + 49.95288825052968, + 49.4963761219487, + 51.35484556698004, + 48.07247639581841, + 47.620312646736856, + 22.25293012809724, + 23.427700473489526, + 24.340036893970193, + 26.04659357254031, + 27.678026884226437, + 28.772764753416144, + 30.545441496187433, + 33.550328607583445, + 36.34223207943441, + 40.4854968211994, + 42.469614741576315, + 49.8473062849506, + 48.96867960067761, + 50.166382245022916, + 49.067151821238554, + 47.96974876108161 + ], + "dft_magnitude": [ + 1164.6950036117196, + 2.359288055252189, + 205.61789806590713, + 0.8010151369478061, + 76.47645522382079, + 2.963085061368112, + 41.5643326103138, + 1.532255790178196, + 33.921466416048034, + 4.434781891206316, + 35.79409045409209, + 2.3209375841975723, + 21.32937075403058, + 3.87824799070102, + 28.62784607642205, + 1.6921761401926523, + 37.129613528131244 + ] + }, + { + "layer": 20, + "mean_snr": 0.792167150953017, + "per_head_snr": [ + 0.7287044345864743, + 0.6785382534712447, + 0.8952848917778549, + 0.8726168689873932, + 0.6847813586748916, + 0.7512416376695624, + 0.7478773427097652, + 0.931332393931674, + 0.7603856625703237, + 0.8632069759588451, + 0.7723227017990278, + 0.7960648088921307, + 0.7542438526664484, + 0.9008657639539048, + 0.6397990795621662, + 0.8455778124148058, + 0.7976678841472494, + 0.7252545389698944, + 0.7027592816112666, + 0.9450771000811635, + 0.8275265685371315, + 0.9067552202814553, + 0.8436038941667826, + 0.8871736632727069, + 0.6644779737334056, + 0.6973872740981973, + 0.9087358648763482, + 0.7038964791279416, + 0.7084681502252976, + 0.7726026031731976, + 0.8918905906652432, + 0.7432279039027508 + ], + "mean_band_energy": [ + 22.806421879324347, + 23.605181837421032, + 24.291673364286737, + 25.40233201972239, + 26.169597886681693, + 27.196646776047935, + 29.40642884392256, + 30.997321316966776, + 34.31664900611043, + 36.18976771811025, + 40.0921717499348, + 43.80549013357276, + 46.272113470596686, + 45.580857281223174, + 42.215643745969324, + 38.56841266308204, + 22.31142393623784, + 24.007133745846566, + 24.763387547704742, + 25.858139726574432, + 26.71324046927581, + 27.147558794146136, + 29.47420432977466, + 31.777742485639948, + 34.062155260531476, + 36.89659850796051, + 40.15948332150374, + 44.34113527477945, + 46.56989904326884, + 45.00298706173712, + 42.27850607041851, + 40.99141645201795 + ], + "dft_magnitude": [ + 1079.2717217203908, + 3.315632791636736, + 167.6591720363883, + 2.4550543055909997, + 57.821915918494525, + 3.771636017184214, + 24.755083352951715, + 3.609376952570467, + 20.996978316445663, + 2.980717863115465, + 19.81697167962629, + 3.3700647268521515, + 17.917294539549644, + 1.2913167566659074, + 19.272959392470415, + 2.243069589006393, + 15.465721869306321 + ] + }, + { + "layer": 21, + "mean_snr": 0.7686863160791969, + "per_head_snr": [ + 0.8964951772397621, + 0.9254569016767132, + 0.6227966005391966, + 0.5997102800822032, + 0.6350372345753799, + 0.9031289045508804, + 0.6519310529208713, + 0.9512503658250386, + 0.8596635561423417, + 0.6751166141994618, + 0.7028601110630174, + 0.6134281725033625, + 0.9107614341778483, + 0.6319496302377519, + 0.646017794069375, + 0.9113823246226411, + 0.8407831398617054, + 0.8930993746448905, + 0.7323022328398331, + 0.6877181871784349, + 0.853963001145126, + 0.8212498157899552, + 0.6081663792263504, + 0.8835656252063471, + 0.6838745042201114, + 0.8958462659277385, + 0.8177649388487404, + 0.6604914479264387, + 0.8323996988244071, + 0.9157993498815752, + 0.6523878995767043, + 0.6815640990100973 + ], + "mean_band_energy": [ + 19.349184839021493, + 20.066099834015972, + 20.8696045838462, + 21.383229384891102, + 22.510736073993236, + 23.155853956414074, + 25.458723033108033, + 27.837170378418914, + 30.199939580598322, + 32.37321893079751, + 36.478172374753505, + 40.45328897637879, + 43.68212610448268, + 44.461193414541754, + 40.56245765819614, + 38.03057065509702, + 18.9932000053115, + 20.438414838664347, + 20.814314690524604, + 21.901133772629198, + 22.476300556488773, + 23.428759758460334, + 25.06036523672266, + 27.160948278345295, + 30.374504854854255, + 32.636352923557574, + 36.43481205611222, + 40.0448043802244, + 44.31087074972207, + 44.486954020549604, + 40.48568715352335, + 39.70186886752814 + ], + "dft_magnitude": [ + 975.620861921773, + 1.6417453879175776, + 179.67388083155288, + 2.439969748578292, + 66.17928919599288, + 2.566903705069901, + 30.500567695112792, + 0.5636556659635596, + 21.04180197156219, + 3.099364833956064, + 20.55890767205079, + 2.5084910893269052, + 21.89219724610759, + 2.4504400351005597, + 23.581227936759042, + 0.27539498520942046, + 19.498862819254953 + ] + }, + { + "layer": 22, + "mean_snr": 0.8163839736440601, + "per_head_snr": [ + 0.7584413767981049, + 0.8574012912058069, + 0.9156655481619104, + 0.626953478254833, + 0.938901197587065, + 0.9172647119408271, + 0.6438528890410845, + 0.7655844160442329, + 0.6477368861973156, + 0.770656830949236, + 0.8866118726347483, + 0.6141946306930035, + 0.8325707013882077, + 0.9487991327175529, + 0.9347137408200368, + 0.9409449677374042, + 0.9159471193135907, + 0.9087089946758041, + 0.7009267748467246, + 0.8878483452600325, + 0.6423807163607369, + 0.9233898402901217, + 0.9772033122901915, + 0.6743753735809874, + 0.8673582363531076, + 0.863475442085101, + 0.9008313533213335, + 0.7209153283413169, + 0.7022647254169317, + 0.9413411621589928, + 0.9014747789756125, + 0.5955519811679658 + ], + "mean_band_energy": [ + 20.95440661491737, + 21.881775826392797, + 22.138948956446878, + 22.93095745025907, + 23.97273769982127, + 25.01831790192774, + 26.159998814803455, + 28.286822130858933, + 29.993808824870086, + 32.75411260588783, + 34.36606251963154, + 35.26930685052146, + 39.37690882493171, + 39.31862198713695, + 37.95555628814754, + 35.0135707273717, + 20.666730960190982, + 21.800140907384403, + 21.92324511352684, + 23.690558195583368, + 24.600207005754953, + 24.92809337061319, + 26.419084862837884, + 28.293238265023263, + 30.363807512734027, + 32.554246608938996, + 34.75599091875458, + 36.001871163486285, + 38.83883637650306, + 40.34120186429673, + 37.357317021430724, + 35.758630491292564 + ], + "dft_magnitude": [ + 953.6851146622781, + 2.2678543082753007, + 129.8608411660516, + 1.3496505886280385, + 48.61436326154344, + 1.6571017468904763, + 29.60052646564474, + 0.8860901831894885, + 13.989546085566403, + 1.1715939592862339, + 12.559690259886217, + 1.4645239258028384, + 18.14787497770152, + 3.8176308422525307, + 15.458570957348389, + 2.4037472472541728, + 13.99781803167241 + ] + }, + { + "layer": 23, + "mean_snr": 0.8679044814409683, + "per_head_snr": [ + 0.9150336190726767, + 0.9288162592454895, + 0.9802210024410792, + 0.7366275444120653, + 0.7959920358824637, + 0.7684941772722645, + 0.9228829253036475, + 0.8977816425617142, + 0.9609681177463695, + 0.9441846692596156, + 0.7574173569589394, + 0.9430057357014393, + 0.898005858387068, + 0.9135898771124457, + 0.911085994387634, + 0.7958290836043962, + 0.9581452378689103, + 0.9019843192693049, + 0.7353688997404091, + 0.8659425212076558, + 0.8290039897959534, + 0.7729687870836804, + 0.9325152934405078, + 0.7278957437124469, + 0.9298023695904093, + 0.7773412631434079, + 0.9010439621825939, + 0.8398963152530892, + 0.9140654944010655, + 0.9119621628243666, + 0.7859124285687294, + 0.9191587186791459 + ], + "mean_band_energy": [ + 17.254644793153933, + 18.40153784991973, + 19.546291018979325, + 19.94473485797076, + 20.756349076767304, + 21.01087949527529, + 22.016077120470793, + 23.163903148514322, + 24.233062312143804, + 25.034720721560753, + 26.378572817983944, + 25.547634364095632, + 27.193541860496453, + 26.742165471952077, + 27.25955713531164, + 26.78880632612904, + 17.40378173090662, + 18.511716561162487, + 19.371895977233912, + 20.03266318847763, + 20.839457952724246, + 21.423767863030683, + 22.168077493926535, + 22.898109285866855, + 23.902604082360888, + 24.969507519073993, + 25.998261947971663, + 25.98083874514997, + 27.075229945184855, + 27.767547005223193, + 27.083867203201414, + 27.026474126081368 + ], + "dft_magnitude": [ + 743.7262789983012, + 0.5859985551228983, + 62.64144667317194, + 1.8161050081115553, + 24.74977681311269, + 1.5757455157371558, + 19.7427869840879, + 0.7634625851606441, + 13.461188201228472, + 0.5922044945163383, + 10.609533796161342, + 0.18310886043956107, + 13.22805558795222, + 1.7950521183948764, + 11.087761489190273, + 2.285493945993516, + 6.763734060666479 + ] + } + ], + "elapsed_s": 0.27177929878234863, + "loader": "raw_hf" +} \ No newline at end of file diff --git a/data/exp_wqk_spectral/SmolLM2-135M_wqk_spectral.json b/data/exp_wqk_spectral/SmolLM2-135M_wqk_spectral.json new file mode 100644 index 0000000000000000000000000000000000000000..602f260349b8a9405f03013631838843bedf04be --- /dev/null +++ b/data/exp_wqk_spectral/SmolLM2-135M_wqk_spectral.json @@ -0,0 +1,2060 @@ +{ + "model": "HuggingFaceTB/SmolLM2-135M", + "short_name": "SmolLM2-135M", + "theta": 100000, + "T_train": 2048, + "d_head": 64, + "n_pairs": 32, + "k_dead": 19, + "n_heads": 9, + "n_kv": 3, + "n_layers": 30, + "global_mean_snr": 0.8460936436935623, + "snr_real_inv": 1.1819022722290766, + "global_min_snr": 0.7506224815587069, + "global_max_snr": 0.9474150560928636, + "layers": [ + { + "layer": 0, + "mean_snr": 0.9474150560928636, + "per_head_snr": [ + 0.9741684990142274, + 0.8824485444339234, + 0.9560061644505358, + 1.1374138595107441, + 0.972491183124939, + 0.9499281664396371, + 0.871836929820609, + 0.9134125017423881, + 0.8690296562987668 + ], + "mean_band_energy": [ + 119.06277021555746, + 106.76171712264983, + 96.45657023416516, + 96.63248183483822, + 116.87230116034542, + 90.86623770282985, + 127.12751758735996, + 113.4697259608741, + 111.97021217839944, + 116.0302302242052, + 110.07808055130626, + 208.16986896023414, + 108.19928949587484, + 131.92156185776406, + 111.998829282195, + 100.65303964459468, + 139.3396422773865, + 107.90821734026778, + 101.42158820385242, + 96.07540317479045, + 112.72147152868057, + 105.13237578439566, + 115.06244897168351, + 92.81924766171521, + 100.52434135373561, + 105.77741583950076, + 141.39882461620022, + 268.8886489376471, + 107.02215779083765, + 126.30767578487838, + 97.14591184154344, + 110.98458124342515 + ], + "dft_magnitude": [ + 3794.800386363734, + 32.27135357364066, + 340.7667440346894, + 117.16439542410667, + 231.99848797653505, + 119.03484248127444, + 325.0490800750393, + 94.75431345716649, + 197.55954547550022, + 60.53775579108324, + 178.82361483431583, + 58.24375526745909, + 273.9010348860763, + 56.522726950245165, + 332.54897328396584, + 61.53072461612055, + 161.9964717854873 + ] + }, + { + "layer": 1, + "mean_snr": 0.8601182726191523, + "per_head_snr": [ + 0.8926854891028884, + 0.8512782289714791, + 0.9346995027825726, + 0.8425628420738289, + 0.8560914491742219, + 0.8365577991828145, + 0.846645725364209, + 0.8468686680592719, + 0.8336747488610833 + ], + "mean_band_energy": [ + 54.13189639493165, + 45.617023903704165, + 61.57732556051272, + 67.33854761813089, + 67.63384500973322, + 71.74503978828938, + 73.54907988749619, + 88.08703937357141, + 82.81720502778298, + 84.99497660130595, + 93.06105061918363, + 75.89008725079809, + 73.39571335199581, + 90.45644743960212, + 96.08251278086507, + 89.7227535639078, + 43.08172439470187, + 49.32898869111929, + 61.957725453697954, + 63.402530973951144, + 80.29305105591558, + 79.90834505299624, + 79.88772041289187, + 87.14358906314293, + 81.84549431933472, + 83.08395062058733, + 97.27601942586232, + 79.75106083735369, + 80.44994393341852, + 91.24765611485003, + 95.2381919243973, + 94.68799742536314 + ], + "dft_magnitude": [ + 2464.6845338713947, + 30.672693920085177, + 217.57859419587297, + 27.255555094908367, + 137.71687343588817, + 27.3337710465692, + 144.81365812022455, + 15.498140751357791, + 107.1708444624343, + 25.89488095488971, + 22.067811875033822, + 22.885487410934058, + 82.1732146429042, + 12.294633251634705, + 30.87133054209525, + 15.60834340886506, + 20.127534765952305 + ] + }, + { + "layer": 2, + "mean_snr": 0.910197795949204, + "per_head_snr": [ + 0.8205300384492341, + 1.0597956562889312, + 0.9362819482003978, + 0.9572627157402278, + 0.7482026838386445, + 0.948131318353862, + 0.9275663949971737, + 0.8767324539669913, + 0.9172769537073736 + ], + "mean_band_energy": [ + 45.69193155775846, + 48.450979748738305, + 57.41193196738363, + 62.06969407765746, + 57.67674936482076, + 63.627626847392875, + 69.89913473631934, + 75.26825013903171, + 74.00692289575217, + 82.0349831001728, + 82.5386407878065, + 61.94059040679495, + 78.68456547351795, + 89.18658108257357, + 84.66497557716609, + 87.00211981350175, + 44.44013150329322, + 51.07785095976778, + 58.63598125748563, + 57.545009749184274, + 65.83326468412588, + 62.939849199332, + 66.64287212891476, + 74.86378440716342, + 73.51798971675329, + 79.2515992365985, + 82.3148943008783, + 68.75816728445346, + 77.63934856236408, + 86.9587421896443, + 86.45429426473846, + 84.94517656375433 + ], + "dft_magnitude": [ + 2241.9746335848404, + 2.6752326022624984, + 200.90036274033142, + 8.865193101548488, + 109.18162755594075, + 11.194491135285768, + 126.73793739387807, + 3.6418259769894985, + 71.62251431979011, + 21.442491674035224, + 28.071586716358183, + 17.2939381867237, + 82.95059759334754, + 19.70268565816639, + 45.4375680147289, + 6.1891404421007925, + 29.867376026683132 + ] + }, + { + "layer": 3, + "mean_snr": 0.8672583993895437, + "per_head_snr": [ + 0.8651988396970556, + 0.8513397992454256, + 0.9367886393032692, + 0.9055163144111744, + 0.9361080038469634, + 0.7826450553577172, + 0.7637701415314625, + 0.8867946774054571, + 0.8771641237073694 + ], + "mean_band_energy": [ + 31.961930953584872, + 37.1590951138838, + 48.84967492058442, + 55.19820058988734, + 59.061536509755115, + 54.70079105401838, + 64.20658442071483, + 64.75061629037992, + 68.79105935157743, + 67.57359130779206, + 65.42657853162632, + 50.57224510846666, + 65.69872318852055, + 75.09501376576519, + 71.11780839562773, + 72.27426081151407, + 31.479099095324234, + 42.378932695395484, + 51.96916515548378, + 59.39878888978137, + 56.01668636568848, + 62.3127990876963, + 59.65786574049417, + 66.07098340481808, + 70.29057343575339, + 68.81184904224529, + 67.73765178451377, + 57.45245591557154, + 63.339342200855754, + 71.64802203158014, + 65.5122171824155, + 69.63985978143127 + ], + "dft_magnitude": [ + 1916.1540021227472, + 21.495784126002505, + 150.29381081002026, + 14.484693379451494, + 103.78081915057078, + 14.23161019949946, + 133.77360153085672, + 12.27348635410198, + 50.34192520345385, + 10.997972325310021, + 37.539419180389174, + 15.881125194937274, + 60.77963544483869, + 11.048196724274431, + 61.12498381016626, + 23.106754372671755, + 33.921007657706696 + ] + }, + { + "layer": 4, + "mean_snr": 0.8777593830903759, + "per_head_snr": [ + 0.8561207987893581, + 0.708857796879682, + 0.8646339984192392, + 0.9465919720252957, + 0.9410688014449393, + 0.8825199299732961, + 0.9088796233336144, + 0.945329388534721, + 0.8458321384132375 + ], + "mean_band_energy": [ + 34.015199529070486, + 35.75370070977753, + 45.01025656482344, + 54.87430468808496, + 49.58098556756435, + 56.23906010929872, + 62.07887532861797, + 68.57085165320501, + 70.89680997643853, + 66.24174531053372, + 74.78868415587193, + 71.25839004150203, + 59.615965813610856, + 76.76763401618899, + 74.5959167468862, + 79.92680395234655, + 35.84432252079856, + 43.325498202441295, + 48.10393919407256, + 49.65260218068416, + 54.393677421132, + 56.94378728972429, + 57.42793245264758, + 64.9356237543444, + 71.09876174934536, + 64.85222951908872, + 68.2948297457358, + 72.43371237063644, + 62.01661608310097, + 77.86503051017105, + 75.73516269633657, + 75.3946933105073 + ], + "dft_magnitude": [ + 1958.5336031645884, + 14.639257799341705, + 210.27260622839833, + 19.523775070521115, + 106.29219047382354, + 12.046958606419222, + 105.47741170965298, + 10.322207589689697, + 90.49958068093393, + 23.382819084647654, + 44.980405819295214, + 11.662971592078524, + 35.87465366119544, + 17.605969041670917, + 64.96428746907465, + 1.6941360399210819, + 71.53773207248196 + ] + }, + { + "layer": 5, + "mean_snr": 0.8818694267362687, + "per_head_snr": [ + 0.759963642005585, + 0.82233190210681, + 0.9209323190423796, + 0.8266433207868437, + 0.8803543633701891, + 0.9527041256025759, + 0.784620318086198, + 1.0998355786658929, + 0.8894392709599439 + ], + "mean_band_energy": [ + 32.00566557316448, + 34.13033251688147, + 37.36048921593566, + 36.18029551192555, + 39.92462962103884, + 46.95959138723859, + 54.13304013060974, + 60.966138265913436, + 59.42627064694276, + 65.68106096698183, + 67.22892119866194, + 57.73913262582735, + 66.30054365224188, + 69.02506959489736, + 60.20621648782804, + 71.42592087357599, + 31.232382840060577, + 36.56769856392384, + 34.193092155486156, + 32.355742241849235, + 35.656132720255265, + 51.50907337961131, + 52.2508533519091, + 62.46262062529609, + 57.94862586029459, + 62.874859528376724, + 65.66218195856663, + 63.21461403660472, + 60.59540564576527, + 66.65664522818636, + 65.5068653609556, + 71.26088466656857 + ], + "dft_magnitude": [ + 1708.640996433375, + 9.126652794295172, + 253.7765692799818, + 5.535526434973703, + 106.7842968128228, + 7.499638632534653, + 49.534080548077135, + 19.882145948704984, + 57.87920043414642, + 13.793604604844022, + 39.20230304729147, + 19.428568133285346, + 72.53276928279618, + 5.727353635342325, + 42.54610025706472, + 17.645849008377354, + 69.37836359394191 + ] + }, + { + "layer": 6, + "mean_snr": 0.850971985805005, + "per_head_snr": [ + 0.8565549275415018, + 0.8129483799147476, + 0.8178387852917338, + 0.7407025991861317, + 0.96890421862519, + 0.7710850835380785, + 0.7525555834322589, + 0.9503042235315625, + 0.9878540711838395 + ], + "mean_band_energy": [ + 30.173465441175875, + 32.66385222517502, + 36.22977834989592, + 39.018794165679004, + 42.84182438267814, + 47.70423326717335, + 53.00737424536664, + 54.2282325331571, + 63.08030311101397, + 65.26623880795745, + 67.4065566788084, + 63.640198832012985, + 75.65584671846257, + 61.88275692844741, + 76.93176962057107, + 74.4251264022846, + 29.351077397375107, + 30.486926291056573, + 32.59073995694545, + 40.84375534113428, + 41.75478392895141, + 48.435966726701935, + 55.337572756421956, + 59.02022773149757, + 62.25813340121143, + 66.63126093008958, + 64.79609876868949, + 71.0771917578921, + 66.83033715765825, + 69.36790169779397, + 80.16956539110122, + 73.9252591805973 + ], + "dft_magnitude": [ + 1777.033150124977, + 13.051772521245317, + 284.2528846894046, + 9.402053940597403, + 124.67496475820172, + 7.122976455591264, + 96.88279312162061, + 3.018553443945752, + 76.5557445531371, + 20.99374547884367, + 72.87089124535419, + 12.12409067574068, + 53.24933910859768, + 20.763896050705178, + 40.15822521925061, + 27.455867221137275, + 20.202695512323317 + ] + }, + { + "layer": 7, + "mean_snr": 0.8755018870549298, + "per_head_snr": [ + 0.9476800237916176, + 0.902841256321888, + 0.9072624145839276, + 0.8909523441210033, + 0.9228492598133415, + 0.7811022961022234, + 0.9296428305137964, + 0.6822789889222486, + 0.9149075693243215 + ], + "mean_band_energy": [ + 27.01188171006336, + 28.065248017671827, + 35.31633287841444, + 33.174606712384474, + 35.79073230119546, + 39.020589702119054, + 44.06272557382402, + 37.5072293128164, + 43.24037805904028, + 42.59101525535349, + 43.28809628641716, + 44.641693082150155, + 53.415287195121664, + 54.630592049091675, + 54.81605303949065, + 44.83209963850835, + 27.072852292162846, + 30.376735342426944, + 33.08381544333635, + 34.486814350514464, + 38.373162602553094, + 39.157642648075985, + 39.84763408777368, + 39.365826115071854, + 42.891155928984595, + 43.263243017498354, + 45.475045400496484, + 45.485443118196045, + 53.15475988285167, + 51.86702601149333, + 56.54022597487316, + 54.465086990484124 + ], + "dft_magnitude": [ + 1336.3110300204553, + 8.953600605860023, + 135.4341215707485, + 7.117973725340785, + 97.41082339332758, + 15.960518425396968, + 52.47227816059192, + 9.992921193224568, + 31.872244032829002, + 21.26767191504674, + 38.98491111049673, + 11.808827992035377, + 25.199681313890657, + 9.353817790801005, + 24.67106619669468, + 5.597893867218622, + 10.44924729274237 + ] + }, + { + "layer": 8, + "mean_snr": 0.8038753346167206, + "per_head_snr": [ + 0.7224274797319881, + 0.8107459986517958, + 0.7546505993743294, + 0.8679977951276275, + 0.8150371869963889, + 0.7870922568473478, + 0.8395904610938237, + 0.8222587914123739, + 0.8150774423148096 + ], + "mean_band_energy": [ + 18.976336867818905, + 19.063323890199573, + 23.308290998785424, + 28.786839224724645, + 32.72069784496804, + 34.88681072862351, + 36.94915617945268, + 45.27466109708083, + 52.758010072825044, + 59.56151666016407, + 63.36496170034601, + 60.575415049117964, + 68.57978923843626, + 62.91007062844114, + 69.20562466399517, + 76.08652486733801, + 18.298995957547323, + 20.823700374617225, + 22.422636135373395, + 28.383054037689167, + 31.705347723080322, + 36.32845686848618, + 39.6005124548018, + 50.928416891753436, + 53.85661544182322, + 58.40796296833345, + 63.99682554958369, + 59.68406817502773, + 66.01442421530896, + 59.320607366993045, + 72.82808187941959, + 69.33328989928097 + ], + "dft_magnitude": [ + 1504.9410256514368, + 11.494663540401064, + 346.92073043386614, + 13.624926370731096, + 121.18336608188392, + 9.732056240963727, + 122.32484611864692, + 3.520618823981671, + 83.47543571861871, + 11.560854373189896, + 70.52912175185499, + 9.839923792267635, + 79.4161451678389, + 9.825589368883339, + 53.380212217133995, + 16.624681826535465, + 35.76841180430506 + ] + }, + { + "layer": 9, + "mean_snr": 0.8304318575056066, + "per_head_snr": [ + 0.8284919039096889, + 0.8383777008242627, + 0.7375917750269121, + 0.6942554408000515, + 0.8422112774951821, + 0.9136776944658217, + 0.8783179430192523, + 0.8511472225878793, + 0.8898157594214087 + ], + "mean_band_energy": [ + 28.289973339864584, + 30.81233103948226, + 32.30514508897131, + 34.23031573242948, + 40.84992067160614, + 48.73403431975599, + 52.94111115656986, + 58.24778221189644, + 59.360402715503696, + 65.1270885054935, + 66.94458761048003, + 53.2371640603314, + 55.154195066604515, + 65.2002658208323, + 67.2191388326832, + 65.69982009563283, + 26.856784416748344, + 28.706289953828357, + 32.61262950411853, + 31.126593730942652, + 43.04369997992808, + 49.092899693012214, + 54.224881225094805, + 58.49477499054946, + 59.18361204826356, + 66.97244097996048, + 65.09438841577013, + 63.79264153066502, + 62.62666161779069, + 64.38928794322435, + 68.48752460095072, + 71.60579214704028 + ], + "dft_magnitude": [ + 1670.6641790460253, + 26.347031264946626, + 259.47445204602894, + 8.229694141015484, + 121.92970492293723, + 12.485626193485412, + 95.59732164066568, + 14.050612036537215, + 66.77122174038463, + 19.29293261485226, + 40.15449349428436, + 11.229701429781118, + 61.23424879222539, + 11.500158874443432, + 36.74815023423679, + 11.280983739455516, + 40.27486646412888 + ] + }, + { + "layer": 10, + "mean_snr": 0.9182860232417914, + "per_head_snr": [ + 0.9242288405618567, + 0.9135417303650927, + 0.9188812523339993, + 0.9115779593940309, + 0.9453328742138108, + 0.9976907640374842, + 0.925012751680061, + 0.8443817912465337, + 0.8839262453432541 + ], + "mean_band_energy": [ + 36.419496964795066, + 39.63179100249363, + 45.187323498694745, + 49.386464685706414, + 59.714075924486984, + 56.37489547857722, + 55.30890440060716, + 56.64505963118515, + 59.374859642925635, + 60.86989178245929, + 64.96590917867587, + 62.22002470708152, + 58.57300888624614, + 63.19441415155961, + 63.64393128240411, + 63.204211342464, + 38.42900310940561, + 39.294558877230784, + 42.95265738795216, + 50.496448098147084, + 53.00768039895658, + 56.881910124205206, + 55.12142365825311, + 61.25696185285582, + 60.16372974215695, + 60.3180328529615, + 61.9165900143092, + 56.211165459664045, + 53.695817032126314, + 63.44143706177917, + 62.68800942826189, + 62.99555638203027 + ], + "dft_magnitude": [ + 1773.5852440406584, + 12.954023824513484, + 128.2485663396096, + 16.935340887601203, + 79.2328314792878, + 18.436543632009354, + 82.25619718042114, + 2.4114103889832257, + 39.40025602283776, + 8.755096301727757, + 20.831599333175358, + 1.7996820490656378, + 32.17439515140983, + 12.123846060416408, + 29.4788682722999, + 12.53816830993672, + 31.260402940143194 + ] + }, + { + "layer": 11, + "mean_snr": 0.9053176295193048, + "per_head_snr": [ + 0.780958959282844, + 1.0008296139753698, + 0.8079381097796579, + 0.940112114394783, + 0.9362126021019624, + 0.9268096044203984, + 0.9468434280345404, + 0.9039470740883044, + 0.9042071595958824 + ], + "mean_band_energy": [ + 35.8346367277437, + 41.55649166863184, + 45.14883869293752, + 54.9554531950536, + 56.2973633440687, + 59.027441821372726, + 60.16647985512888, + 62.36746473752237, + 55.09809147191832, + 60.11874747285815, + 63.724171708511626, + 54.56920870478224, + 59.840574916417054, + 61.3893131033573, + 64.04722577232153, + 57.07049729666167, + 36.283942342864165, + 40.460820709049734, + 47.959596691300554, + 51.49299664092392, + 55.51448944458861, + 54.21701958880198, + 60.04191013253686, + 57.94073950107648, + 60.85486287673934, + 62.89746015273608, + 65.2867478957728, + 54.77579877357685, + 56.98053773678723, + 63.323251815523285, + 64.1157695257811, + 64.58057756674485 + ], + "dft_magnitude": [ + 1787.9385218840912, + 13.326591041436775, + 117.00298075316068, + 5.747554864698602, + 81.75813891040285, + 21.579513518366415, + 76.0246289705402, + 6.241281491496462, + 55.775281455422544, + 7.377701340242565, + 7.681630570130825, + 3.3585687642882713, + 39.00794078165403, + 8.845765919555216, + 30.939756022767867, + 23.48195785221751, + 13.548043613255118 + ] + }, + { + "layer": 12, + "mean_snr": 0.8606241220208308, + "per_head_snr": [ + 0.9000249065361888, + 0.8169154644158375, + 0.8308289769056332, + 0.8973118760959083, + 0.8276955980277381, + 0.9491346275097581, + 0.9255616160497211, + 0.7904499331085223, + 0.8076940995381697 + ], + "mean_band_energy": [ + 24.958876044106546, + 25.818114226686728, + 30.10753630147854, + 35.344910620461405, + 42.31045701591222, + 47.611583778742215, + 52.53319578100278, + 56.42655958320903, + 59.38856794899665, + 59.37635641574813, + 65.05292378329366, + 65.2901987460513, + 49.64944277691747, + 66.6071235882661, + 65.24441507034969, + 63.06280518424339, + 25.321714103239174, + 27.70148910393368, + 31.844736105461227, + 36.50856734782475, + 40.06532174310728, + 49.767295525347144, + 54.094660622908776, + 54.182082295234444, + 57.98195974198052, + 60.58268491686527, + 64.87009705860487, + 55.86901734172776, + 52.49858558603622, + 64.94543380968015, + 61.62584477238589, + 60.351347838636634 + ], + "dft_magnitude": [ + 1606.9939047784396, + 16.95564187700947, + 245.64264485962934, + 2.6188533660418156, + 113.12277591849275, + 3.169839969331956, + 92.87146668957382, + 4.405504908769809, + 77.2297147303185, + 21.96223890752905, + 26.82963403736959, + 10.781963201464615, + 27.408259282691535, + 10.625188488567614, + 52.05932566360392, + 12.442925029481975, + 51.8972358668766 + ] + }, + { + "layer": 13, + "mean_snr": 0.8571580689817755, + "per_head_snr": [ + 0.869694139074382, + 0.9175118373486024, + 0.8876034569051425, + 0.8508597742222894, + 0.9035796007696811, + 0.8403748603895478, + 0.834472376699509, + 0.815580444694068, + 0.7947461307327559 + ], + "mean_band_energy": [ + 31.065531926129033, + 33.800150096704456, + 39.941801372656755, + 44.469267272939334, + 52.04404345083463, + 57.8131469276456, + 58.26313687402191, + 66.53558468652766, + 67.79105691283738, + 68.74631763143037, + 69.57841533359891, + 72.86541969794699, + 61.875219332178276, + 68.12110225709911, + 69.36373167594462, + 63.18703367124964, + 29.687745475219295, + 34.153736098947775, + 41.98556946421981, + 51.82210085932885, + 54.833936847795634, + 56.82227488130926, + 63.09920314507766, + 63.68113356732968, + 63.98085715136391, + 66.25726310036276, + 72.3836841844428, + 73.55225552585476, + 65.06851483034819, + 69.33413860747413, + 61.397581136620225, + 66.69898509810021 + ], + "dft_magnitude": [ + 1860.2199390935396, + 10.728963166377541, + 241.8713892768054, + 14.620146003443388, + 98.6367076948581, + 18.261271411298384, + 72.23528051724674, + 16.86112142094896, + 68.90645450592415, + 9.514861286906083, + 34.972705285768775, + 19.406513241971417, + 26.630182981930854, + 15.548402274872041, + 41.58141741688591, + 9.006822140814663, + 55.499880866961576 + ] + }, + { + "layer": 14, + "mean_snr": 0.7756414177821299, + "per_head_snr": [ + 0.8465518260715557, + 0.718906288047092, + 0.8114026086103093, + 0.7498591257180703, + 0.7837011121218286, + 0.7797784478198776, + 0.7679147377241428, + 0.7828115026513416, + 0.7398471112749511 + ], + "mean_band_energy": [ + 12.298556925417776, + 13.662197698780712, + 16.856819439839644, + 20.984855713857073, + 25.196304263383393, + 30.392930465994837, + 34.12114000945616, + 41.017528818537045, + 42.48326131840625, + 49.22459743829833, + 53.80461404487679, + 52.863258075975786, + 65.93790511326301, + 74.38471624824079, + 58.14932136095225, + 47.558764114573506, + 12.01914401099602, + 12.924087792242359, + 16.415896839406603, + 21.655554444424055, + 24.429122128867423, + 29.175222578639037, + 34.60454445893437, + 39.1887337151538, + 42.38974417396665, + 49.115264047559826, + 54.44709768135208, + 56.421029607080726, + 69.28587245278676, + 69.84337808444565, + 59.98093564295485, + 53.77997532849387 + ], + "dft_magnitude": [ + 1284.6123740371575, + 11.06510034762822, + 368.4626836557224, + 4.157397924679142, + 164.62610407167546, + 8.011479604877788, + 84.12209005853711, + 12.632066791558282, + 34.66704544270487, + 11.79825617726236, + 26.36241446970897, + 14.768752589787907, + 51.500144562074205, + 3.398905935825243, + 38.806723808270156, + 4.341632800690644, + 39.77181430743735 + ] + }, + { + "layer": 15, + "mean_snr": 0.7945046804224326, + "per_head_snr": [ + 0.8130236971874312, + 0.7698110655320908, + 0.8693489244407053, + 0.8449776637264317, + 0.8244812022529432, + 0.8196509095280836, + 0.6991166378544413, + 0.7452490354770549, + 0.7648829878027116 + ], + "mean_band_energy": [ + 12.32379702420773, + 14.346152275634571, + 15.917932320101322, + 20.162531042544273, + 23.069882963094017, + 31.24085557135201, + 34.90581503943338, + 40.60095273898181, + 45.77978518605099, + 54.1745330390791, + 61.60129541175856, + 69.09063565350624, + 61.28508023250293, + 65.89577821197109, + 63.33485418076609, + 63.56437741043457, + 13.079304106963933, + 12.366317982496103, + 17.100109730113914, + 19.85118002264763, + 24.08641816170122, + 29.836258180972436, + 34.72400635278547, + 41.976608852657314, + 48.22846219603566, + 53.73208950401701, + 60.35132419986818, + 64.84863044837464, + 60.125631212750605, + 64.77060533510955, + 68.26139859926752, + 55.23667393321325 + ], + "dft_magnitude": [ + 1345.8692771203932, + 8.697676308952136, + 401.6096336761142, + 5.689497569446667, + 127.14797196259677, + 8.758983445906948, + 89.99133565433301, + 15.134254548558774, + 83.97445810964234, + 11.606125480511015, + 56.43777952561285, + 11.959696379480018, + 31.237867879670283, + 13.441022825054688, + 39.76927737324845, + 13.820129849295343, + 57.51908328559023 + ] + }, + { + "layer": 16, + "mean_snr": 0.8578323716344208, + "per_head_snr": [ + 1.0374067640584967, + 0.7948484132944815, + 0.8454386680322561, + 0.8210228411774434, + 0.8846234859605516, + 0.9001431359638518, + 0.7771964619307336, + 0.8634164349050492, + 0.796395139386924 + ], + "mean_band_energy": [ + 21.768371060941295, + 26.685914853852992, + 37.610427150759655, + 41.174526808215205, + 46.35452822753294, + 52.29828842227092, + 54.899342927864964, + 59.98168032093534, + 60.072860722929214, + 65.80772207473852, + 68.53844095402458, + 66.18923175598619, + 64.38329193574133, + 67.47218731730861, + 62.061368024333284, + 68.1599691851504, + 19.983174148796518, + 28.530491326928388, + 37.762399631441866, + 40.667615066832546, + 47.82927490445523, + 49.18931101372414, + 55.45228275308791, + 61.904533149634794, + 61.43586446319452, + 64.7935466631215, + 70.66187649018288, + 63.23377594754402, + 60.966947713139724, + 66.31232210202319, + 58.355653068747706, + 66.28335782866391 + ], + "dft_magnitude": [ + 1716.8205780141043, + 10.131420195424061, + 263.50227856741145, + 10.666969826240944, + 102.07739784357915, + 6.267433819346848, + 88.42629893463268, + 4.077488542495164, + 77.94160179939772, + 3.5810271664123685, + 43.455539553714885, + 14.251558276831018, + 73.4258067665713, + 5.135436038506131, + 58.996331914980814, + 6.186368824801173, + 60.5483696597571 + ] + }, + { + "layer": 17, + "mean_snr": 0.8101025274383727, + "per_head_snr": [ + 0.7709727815602914, + 0.7968555349335661, + 0.7818467019735404, + 0.7415409005238723, + 0.7590117447975389, + 0.8175101794088887, + 0.8025637047356642, + 0.7549438788554698, + 1.0656773201565237 + ], + "mean_band_energy": [ + 8.78196289206829, + 9.924162627821651, + 13.669549992344887, + 15.304834188955986, + 18.333652695606002, + 22.939638315025263, + 26.229001909845618, + 29.000040667397936, + 34.97009481217393, + 38.14320262633882, + 41.940002861800245, + 53.260765779801304, + 76.68972885358029, + 82.82071375444815, + 77.40178903238845, + 67.43580941393685, + 8.237306271975692, + 9.266396595107684, + 13.935609015505626, + 15.659544997286876, + 19.304939656460697, + 23.540295690893565, + 26.601189051064075, + 28.559087101078806, + 33.70499683990514, + 37.04403064126846, + 43.21405197038884, + 53.306626476114076, + 73.09717335296627, + 78.68018143234056, + 78.18796979637477, + 52.627025842259144 + ], + "dft_magnitude": [ + 1211.811375154524, + 21.202387609306097, + 446.9674542325432, + 13.077812170606835, + 255.69841044509877, + 14.160835711170902, + 123.4610471927866, + 7.266797676322445, + 49.73340950600039, + 16.758706839936256, + 60.972491519462785, + 18.14112575686224, + 51.4244072782623, + 16.897871898003388, + 40.98314967364774, + 16.921178494649684, + 23.213337145626383 + ] + }, + { + "layer": 18, + "mean_snr": 0.7506224815587069, + "per_head_snr": [ + 0.7796139293728802, + 0.8702418737221268, + 0.7230736383285485, + 0.7702348010948816, + 0.7379186433905739, + 0.8464757065161376, + 0.6797791010752329, + 0.6767652930927179, + 0.6714993474352628 + ], + "mean_band_energy": [ + 7.667078549571859, + 8.83561684218082, + 13.044019392813489, + 14.271654114290136, + 19.109605215911017, + 19.9213038815652, + 22.968822861402188, + 25.526437691448294, + 31.28009218833153, + 38.544042036314195, + 42.530849269278406, + 56.16430420812074, + 69.25546082180936, + 72.381484610142, + 69.17397862901973, + 49.39122302471636, + 7.697641951086477, + 9.08126964473657, + 12.663552807673774, + 15.207726817604389, + 19.26909751713937, + 20.238111856293166, + 23.217542043626977, + 26.93911767556489, + 32.34546001260916, + 38.412825917523634, + 44.87291080807649, + 55.39315734130244, + 69.03026400939451, + 72.94142830024725, + 71.65624302247649, + 57.23986162862684 + ], + "dft_magnitude": [ + 1136.2721846908976, + 12.015036509048603, + 427.26511277190303, + 9.013947679432622, + 211.93723981543445, + 11.060732209557354, + 104.29185927123491, + 8.495507987686208, + 48.67250453152348, + 7.260485365644009, + 53.25492549843342, + 6.4413265380494975, + 46.46719993509457, + 10.209137528585147, + 29.873579710775207, + 3.23116617388724, + 24.706946490456062 + ] + }, + { + "layer": 19, + "mean_snr": 0.870433600435796, + "per_head_snr": [ + 0.8126280610018042, + 0.8605184257920494, + 0.8669401730384217, + 1.0090095203972853, + 0.7898693072236544, + 0.752405639985508, + 0.8314337177639682, + 0.93830800247457, + 0.9727895562449016 + ], + "mean_band_energy": [ + 15.387086474311193, + 18.59949704678383, + 23.195246032897746, + 25.822097836313308, + 31.92626736058037, + 32.594777617906594, + 35.08600117818898, + 36.636815568655415, + 38.41226088547033, + 40.2536581554359, + 41.06595419419106, + 43.45399269407421, + 43.580022537799614, + 34.60200044553895, + 39.94077680799241, + 38.749733904270016, + 14.44868136312175, + 19.180690327660738, + 24.565662548633423, + 27.218177707117167, + 28.008399710352577, + 29.003099127293392, + 33.56303733420246, + 37.0464992804827, + 38.06312260699345, + 40.97160083507578, + 39.39069893237896, + 42.32857604708175, + 41.263481820991984, + 37.23567209117047, + 38.72798273333766, + 35.96618568736873 + ], + "dft_magnitude": [ + 1066.2877568936728, + 8.74380190909098, + 157.20253416603285, + 6.914512724978381, + 55.28986785755228, + 9.784062236716972, + 39.68312592404113, + 8.142618691874416, + 42.51273297761929, + 11.010749999825276, + 40.13564146721596, + 3.2385944763904146, + 35.09931733192986, + 5.8071739564613205, + 14.96635018747653, + 6.661332771090153, + 13.038391850785047 + ] + }, + { + "layer": 20, + "mean_snr": 0.7623364168805327, + "per_head_snr": [ + 0.7786893761050652, + 0.8765680693456402, + 0.7916665529366907, + 0.6995143300564244, + 0.6962420653839342, + 0.7292433650667125, + 0.6970874452360961, + 0.7711543527700474, + 0.8208621950241839 + ], + "mean_band_energy": [ + 8.827136934700812, + 10.610866073932158, + 12.131424688613796, + 14.680029124879958, + 16.814493884793105, + 20.577350057124004, + 21.724693979668903, + 27.757533932933157, + 30.328300560402706, + 35.44628960189329, + 40.53713727248319, + 51.0958946823061, + 55.640018813550014, + 59.63692597275167, + 50.32696172239892, + 54.08202210540882, + 8.437093120535117, + 9.699049218273135, + 11.401905832540027, + 15.145253810349187, + 17.19527337811936, + 21.04124716190733, + 23.485629272274966, + 27.701544200912583, + 31.138335669098613, + 36.68012077523377, + 39.663420265604515, + 52.02790714120733, + 54.98132685955458, + 61.42545703756238, + 59.89583251851179, + 48.576078476252356 + ], + "dft_magnitude": [ + 1028.7125541457776, + 8.866135618559527, + 344.01790206029665, + 5.261971540815643, + 154.14403165519195, + 8.565738575691999, + 71.5138014161825, + 12.053489332452793, + 50.737830672569494, + 12.881868680498618, + 48.750600150125365, + 14.670447292833686, + 37.50216081621284, + 15.54225331298027, + 39.89371271108289, + 11.334256230475987, + 63.65458460007676 + ] + }, + { + "layer": 21, + "mean_snr": 0.8109439371657807, + "per_head_snr": [ + 0.9090654733786576, + 0.7354247635152567, + 0.7623035655927212, + 0.8043068081642752, + 0.890672704002515, + 0.8260492749139321, + 0.8744229359980019, + 0.7729642503015871, + 0.7232856586250797 + ], + "mean_band_energy": [ + 12.397040832920084, + 12.016383832101205, + 14.768120636748355, + 17.94721872372275, + 22.324872308784002, + 22.787939772422003, + 28.247063951325725, + 31.878431330146995, + 38.389871590271774, + 40.00135460642289, + 48.86975921597044, + 49.63831171156249, + 52.428744528281996, + 52.09352067870653, + 48.464111233486875, + 43.99969698051066, + 12.162834716761418, + 15.188314900286022, + 16.22713750543708, + 17.088075233506377, + 21.033975750505874, + 22.80227351634638, + 25.490047622763615, + 31.85590241464955, + 36.753117559684625, + 40.2840180690608, + 47.39230916117307, + 46.34636995463815, + 53.94693551030837, + 53.94801450380745, + 51.53092031408245, + 48.11594915101253 + ], + "dft_magnitude": [ + 1076.4186378174086, + 6.971598654415447, + 298.1945629094486, + 11.812107966505927, + 98.57929832690472, + 14.114758899865198, + 64.19960523510579, + 5.572197139365756, + 42.0177650947216, + 6.918977381445049, + 30.989970614632167, + 5.767454305427744, + 42.66523156807224, + 6.607728970371197, + 41.040465768390334, + 4.21642778364241, + 15.564912940396994 + ] + }, + { + "layer": 22, + "mean_snr": 0.8204875078983362, + "per_head_snr": [ + 0.7789974606134763, + 0.9748866206384431, + 0.7818018591848989, + 0.8967190518635811, + 0.7184119199676511, + 0.8082141178962413, + 0.8219629026819911, + 0.7530519180897836, + 0.850341720148959 + ], + "mean_band_energy": [ + 12.19751067718203, + 14.451874938198468, + 17.4716357614336, + 21.650676734635553, + 23.670511919093315, + 25.614791648917944, + 29.831897153552493, + 33.35911839209348, + 36.474401667460874, + 39.559037224344515, + 43.804752193533076, + 48.40679069867857, + 50.76271357959139, + 48.2285293493106, + 51.87518146443175, + 51.226772745612124, + 12.598588527218615, + 14.324555579253898, + 17.895801406866827, + 20.380938189069898, + 23.176214610644404, + 25.61244863696821, + 30.959207992198042, + 32.79283311065218, + 36.04917540262313, + 39.695999682612964, + 43.035298146335435, + 46.568672469398194, + 49.99522149230685, + 49.68144802933724, + 48.80169246360455, + 55.0202625524057 + ], + "dft_magnitude": [ + 1095.1745544395658, + 3.2068919706985555, + 254.44509308909426, + 4.424584057353315, + 108.00891806925154, + 2.68391478065292, + 67.72218113835731, + 3.1641589830929786, + 65.0414777255403, + 6.597556047266157, + 54.08313030001729, + 6.777605007826068, + 48.5648568606757, + 6.5724081685447535, + 44.68885388349672, + 9.816403665910553, + 37.97494552341311 + ] + }, + { + "layer": 23, + "mean_snr": 0.7642259609095428, + "per_head_snr": [ + 0.7508865722374093, + 0.7455356523077566, + 0.6954179396384899, + 0.8166173919430825, + 0.787760794031573, + 0.7867720145715379, + 0.8526950247884716, + 0.7126518223088679, + 0.7296964363586964 + ], + "mean_band_energy": [ + 7.089070516839064, + 9.39011650522385, + 9.748802441789545, + 12.937171802357776, + 14.010931964268126, + 16.234534290214203, + 20.310860734397313, + 26.432259018284146, + 31.5837874022064, + 35.93341640835019, + 42.88869310788896, + 47.50424711456354, + 51.18823401356003, + 50.067118747626736, + 49.779238339326334, + 42.72915173226933, + 7.90570811377307, + 7.993818011194217, + 11.416026711374993, + 11.931024166217115, + 14.517697627270799, + 20.318024278081364, + 19.623815457114787, + 26.55689321533076, + 29.934276142738653, + 33.013896006042515, + 45.98603664446505, + 49.548016107357704, + 52.70298926158315, + 52.4743053149532, + 42.20993044779713, + 46.781087286178135 + ], + "dft_magnitude": [ + 940.7411789306382, + 3.4765165784418457, + 329.79000636998376, + 3.23251794461914, + 107.82562132958381, + 14.708381367749128, + 51.866260403135776, + 9.296473807892898, + 51.10389597909642, + 8.344162528745757, + 40.27968233547051, + 9.783379255726535, + 44.69300252034142, + 18.79219919346999, + 41.34617464483692, + 14.190592199019722, + 38.948981077851386 + ] + }, + { + "layer": 24, + "mean_snr": 0.7991359000414157, + "per_head_snr": [ + 0.7760223819683884, + 0.724990727315991, + 0.7676655698421239, + 0.9687700868000848, + 0.756901810324911, + 0.8149131388318651, + 0.7659120581768044, + 0.8039141264968618, + 0.8131332006157107 + ], + "mean_band_energy": [ + 9.227900540311275, + 10.053242325411267, + 12.536655728229713, + 13.051578378077132, + 17.736803062267654, + 18.116064836860247, + 20.14856334963397, + 23.609728646753386, + 26.303817951230585, + 29.32280481527487, + 35.08973710666904, + 37.95533595871757, + 49.82524566457065, + 46.25078593790343, + 42.764483648325594, + 37.171171863490926, + 9.297165140342761, + 9.833574337289361, + 11.131788663826443, + 13.186998628148167, + 14.679825405432906, + 19.702770190046465, + 22.729188484845963, + 22.779544370103167, + 27.0841659471533, + 29.608311607297512, + 36.05553501887102, + 39.70157868226584, + 49.00984839533034, + 42.66071219375661, + 38.63263187495992, + 39.03673891372377 + ], + "dft_magnitude": [ + 854.2942976671209, + 2.3732643740192474, + 253.24134390946062, + 11.371765783145511, + 110.72991268885302, + 6.888917228105026, + 36.896737161214105, + 11.377270518576587, + 26.31037928034323, + 4.5750901187623265, + 39.24495543643863, + 7.874202564539857, + 42.516634486343634, + 9.09183406232236, + 33.12774410843132, + 1.6843742504742767, + 9.787585703118566 + ] + }, + { + "layer": 25, + "mean_snr": 0.8963458875083652, + "per_head_snr": [ + 0.856376576563191, + 0.864023962802613, + 0.9153271251803149, + 0.8513239477646946, + 0.9521562959597869, + 0.8821228534650308, + 0.930861399295812, + 0.8152056104664147, + 0.9997152160774287 + ], + "mean_band_energy": [ + 17.952215347331883, + 22.637901111749986, + 23.836681441306308, + 26.982941845729037, + 30.446277422943467, + 32.64449227502337, + 34.24217304986556, + 36.12254381411721, + 36.27960517910555, + 38.07873112764115, + 39.64561539232656, + 40.39553823727503, + 39.073032517218, + 41.8414394966711, + 38.28554606152405, + 41.04305405618319, + 17.932703004726235, + 20.806321443435586, + 25.339908643807096, + 25.771069599675634, + 28.669480159190446, + 32.610064841688505, + 34.43279855999718, + 35.939271312844006, + 34.91995200290474, + 36.91788247729856, + 39.66078019708834, + 38.32805352949453, + 38.941107127632854, + 40.22572845633928, + 38.91793779700561, + 37.12871880431688 + ], + "dft_magnitude": [ + 1066.049566333457, + 8.193037983992987, + 126.71265064313788, + 3.381223691375353, + 63.19293107402993, + 3.6329264496892595, + 34.29130187347302, + 1.2349925791838305, + 34.105784405789834, + 8.44538342267688, + 21.25568106776524, + 5.982575263835063, + 28.090894406912042, + 7.492201121686651, + 24.80587931385922, + 5.703325997333402, + 28.897938525509062 + ] + }, + { + "layer": 26, + "mean_snr": 0.8188976672977325, + "per_head_snr": [ + 0.9021495666142174, + 0.7208632486347926, + 0.7766547945755292, + 0.6976694011159285, + 0.979421292846843, + 0.6666083411521924, + 0.8333121931535894, + 0.8995822739125068, + 0.8938178936739933 + ], + "mean_band_energy": [ + 10.693780002741866, + 12.780554619494634, + 16.80400187086913, + 17.602869094586467, + 20.320165760418686, + 23.632400113375233, + 24.25353036560466, + 27.767543291407314, + 27.939312534788908, + 30.45365746904584, + 32.05391927031724, + 34.91036560687631, + 35.60770085266828, + 36.46601478054825, + 37.41931911546885, + 33.385493774401844, + 11.627444569614344, + 13.920329463995195, + 14.935249016258373, + 18.244257062144136, + 22.702389166644934, + 22.394802806501882, + 25.536597951174674, + 26.97100672084552, + 30.490855348998796, + 30.9405213887228, + 31.278801260425627, + 33.8728501676941, + 35.05526951324883, + 37.59090145742155, + 35.364393833007156, + 36.30848890128724 + ], + "dft_magnitude": [ + 849.3247871505987, + 2.7999334314916484, + 154.66755869808608, + 3.678476961737016, + 76.409386747901, + 3.352960647555538, + 47.055958009781804, + 4.699724855781606, + 31.221489671909453, + 6.276275121239165, + 29.385502703958483, + 8.657964321613404, + 25.97072334811691, + 9.223392552884809, + 23.873411213305054, + 7.243304254802261, + 25.159326286097894 + ] + }, + { + "layer": 27, + "mean_snr": 0.9016465773784194, + "per_head_snr": [ + 0.8239099788494628, + 0.9665077993282405, + 0.7331394830960026, + 0.8932662497954683, + 0.9996276152285483, + 0.8948827342447108, + 1.0287140841695575, + 0.8408770639008363, + 0.9338941877929474 + ], + "mean_band_energy": [ + 16.441844782709236, + 19.525791823162365, + 21.17585791753398, + 24.028158763637812, + 24.79502381254215, + 26.911322505064845, + 28.645916390397538, + 29.268518935983973, + 28.682750336502245, + 28.735866087892873, + 30.23839774846637, + 31.7990537587757, + 31.379929678669605, + 32.95081466909505, + 31.59064920722254, + 34.08569571325178, + 16.16967231262341, + 18.84855292681305, + 19.303910586690122, + 23.786663755329855, + 25.57185791376143, + 27.129516832425526, + 27.15674844683569, + 29.226101105035063, + 28.358498207851955, + 29.66177930116833, + 30.452610837468278, + 30.625387184997635, + 33.31197768441752, + 31.049845914857492, + 31.136274930636823, + 29.52705260683039 + ], + "dft_magnitude": [ + 871.5720426786506, + 3.77458143765722, + 82.29152341535868, + 6.480121573489194, + 51.87797266349698, + 7.261194718562865, + 24.89172956730269, + 8.213795614276684, + 23.066762351175015, + 1.141329751682165, + 16.30967794602235, + 0.7368625672257748, + 19.71681079731259, + 6.528793531277551, + 18.313914995862845, + 8.976881729952524, + 22.748201089992847 + ] + }, + { + "layer": 28, + "mean_snr": 0.817546095151311, + "per_head_snr": [ + 0.7626707124189251, + 0.7351422922469082, + 0.7523536403813107, + 0.8803053577066697, + 0.7941461685436298, + 0.9367741356569278, + 0.7683260660389951, + 0.9035966106738512, + 0.8245998726945819 + ], + "mean_band_energy": [ + 10.566010221487169, + 10.82626337162632, + 13.026011909379191, + 14.965982079948365, + 15.479577954144917, + 16.653868112393443, + 17.817147385783546, + 20.278201757697655, + 23.529153126910046, + 25.429367318075744, + 27.48865439955436, + 30.575308959206318, + 34.79795521306697, + 36.10776824652526, + 34.84413619907814, + 32.405291524385774, + 10.816134445464195, + 12.154447416010177, + 12.522319454779014, + 15.529252221856975, + 15.07947683020054, + 16.73244814856154, + 19.847135479453033, + 21.707631106767888, + 23.788990899058653, + 24.502692458787717, + 27.054858720527186, + 31.009094380132563, + 34.02992323077415, + 38.28122849523808, + 30.02719437436512, + 34.80189234366834 + ], + "dft_magnitude": [ + 732.6754177849084, + 3.8905542581258166, + 166.46830891459877, + 3.074032910147873, + 73.305229832063, + 3.6405840035129633, + 40.95711988168784, + 6.528854788375047, + 25.202043930482866, + 2.944387353255923, + 24.22626070294639, + 8.167240043079412, + 23.10104614644125, + 11.084280675170794, + 30.61680071758129, + 8.696416726275277, + 31.24605809685596 + ] + }, + { + "layer": 29, + "mean_snr": 0.8853210386802035, + "per_head_snr": [ + 0.8776279618539956, + 0.9594307784380203, + 0.842492707783484, + 0.953870589651571, + 1.0026496817917658, + 0.721573249741019, + 0.9029429315444268, + 0.8628606562955874, + 0.8444407910219616 + ], + "mean_band_energy": [ + 13.863476435882806, + 15.49563006326812, + 13.96392760243931, + 14.59783139478454, + 16.416650793369705, + 16.643095370596996, + 18.555739322549243, + 19.560260896508098, + 20.93795459388103, + 21.906080988494903, + 26.32385728652783, + 25.65735368036298, + 26.750812949532854, + 30.066071759025288, + 29.76724405369992, + 29.034370314278362, + 13.27045503945968, + 14.405009793266332, + 13.996601960183686, + 15.03566181193684, + 15.64918527476351, + 16.939398516304962, + 18.26464150417544, + 19.75464314730195, + 20.949515339731825, + 21.888245240084263, + 24.2376826664856, + 25.8884804987833, + 27.420798612765427, + 30.530321452374537, + 29.35287245465439, + 28.65998126667604 + ], + "dft_magnitude": [ + 675.7838520841497, + 1.952032502747169, + 111.76028773094183, + 1.9039387624593125, + 47.72646846352044, + 1.9142363396128732, + 27.158354222781572, + 5.165549461107223, + 21.798541905036032, + 2.4564493354123447, + 11.63453789895647, + 2.9645686763065493, + 16.190344488279546, + 0.5360294599447212, + 21.385134863045757, + 3.2297275678076223, + 16.3410203039453 + ] + } + ], + "elapsed_s": 0.09852337837219238, + "loader": "raw_hf" +} \ No newline at end of file diff --git a/data/exp_wqk_spectral/SmolLM2-360M_wqk_spectral.json b/data/exp_wqk_spectral/SmolLM2-360M_wqk_spectral.json new file mode 100644 index 0000000000000000000000000000000000000000..a2e3d60e265ea7edd30c82d6fdba3add031675c7 --- /dev/null +++ b/data/exp_wqk_spectral/SmolLM2-360M_wqk_spectral.json @@ -0,0 +1,2388 @@ +{ + "model": "HuggingFaceTB/SmolLM2-360M", + "short_name": "SmolLM2-360M", + "theta": 100000, + "T_train": 2048, + "d_head": 64, + "n_pairs": 32, + "k_dead": 19, + "n_heads": 15, + "n_kv": 5, + "n_layers": 32, + "global_mean_snr": 0.8514445080773414, + "snr_real_inv": 1.1744746610182661, + "global_min_snr": 0.7725419482918956, + "global_max_snr": 0.9165189463594561, + "layers": [ + { + "layer": 0, + "mean_snr": 0.9120765686762167, + "per_head_snr": [ + 0.9814810690219082, + 0.9330083231883257, + 0.8260352920579319, + 0.9231561295214272, + 0.9390358539689981, + 1.1118609334970107, + 0.9685282178124476, + 0.8605527921371081, + 0.8828254188377935, + 0.8846155130337885, + 0.8332609944975763, + 0.8360274221743257, + 0.9021823003120258, + 0.9237931512530456, + 0.8747851188295376 + ], + "mean_band_energy": [ + 49.718416045330315, + 51.323867778550024, + 44.64397894848401, + 39.54275951802943, + 56.20181808568291, + 42.53624489765164, + 49.398978116441896, + 51.24070404563005, + 53.55581209480888, + 52.223675125285325, + 69.88118553320558, + 54.602986708211056, + 83.54997935023142, + 59.70481925839705, + 62.77702719676747, + 67.76019580643515, + 67.84952519573213, + 42.658350662568196, + 42.89979078222271, + 49.206384648322306, + 58.60555784872252, + 41.83712451531171, + 43.35236353287702, + 48.18743668418795, + 50.25799843277352, + 53.70005494836905, + 72.2998609897831, + 73.4646599829789, + 78.09343727243055, + 64.30624644220713, + 51.95193532627456, + 80.43246701583321 + ], + "dft_magnitude": [ + 1807.7656427897368, + 8.651529110467543, + 198.7418755031362, + 29.39604424446086, + 39.67017946642641, + 33.28381890926638, + 60.561532005576, + 0.3510711335002615, + 82.63288574911894, + 37.69240894757921, + 29.576351028770205, + 25.707565463212354, + 86.73324546834341, + 36.68912761246087, + 43.74976031738748, + 67.61454247251231, + 62.30968671380049 + ] + }, + { + "layer": 1, + "mean_snr": 0.8829695407845616, + "per_head_snr": [ + 0.753833488599752, + 0.8332416845077183, + 0.8825734748323293, + 0.8794639534724442, + 0.8937669105567635, + 0.9256977440852957, + 0.9715462545982678, + 0.9145844807893699, + 0.9155891686822051, + 0.7397476864494159, + 0.9503754337087564, + 0.8502495753456782, + 0.9257732778343754, + 0.9090549433309822, + 0.899045034975069 + ], + "mean_band_energy": [ + 70.82353688815283, + 67.54892923908868, + 94.2973051063039, + 90.58573765832304, + 94.02666590888411, + 122.97470416914172, + 119.4525333190192, + 121.05217218493773, + 127.52483880431623, + 123.80722769147908, + 135.58853717206068, + 109.07563282433772, + 137.43571358709283, + 128.28937951773142, + 143.3280216078825, + 144.82486767762978, + 63.010654068045234, + 74.12182225427075, + 92.62959005462001, + 106.99080213741165, + 102.43508497257677, + 97.96856593851437, + 119.57471602089612, + 124.21010024814592, + 127.30961071465559, + 126.56492851876519, + 126.85745162558415, + 107.71731766028309, + 131.56074183522117, + 124.01230009884912, + 143.1324773178714, + 146.45682805705823 + ], + "dft_magnitude": [ + 3645.18879487915, + 19.949259309385877, + 328.82550927440127, + 10.604659577636918, + 225.062212929446, + 48.956493270785415, + 193.99164627097696, + 41.32782837070804, + 148.0148320204276, + 31.76208135241469, + 127.54814383700638, + 40.77800005162278, + 125.57057632795767, + 49.77938490993386, + 102.66293894119937, + 4.718485504537482, + 12.786163127215332 + ] + }, + { + "layer": 2, + "mean_snr": 0.8647236474922109, + "per_head_snr": [ + 0.7604339833021876, + 0.7156522330604541, + 0.9579470370936196, + 0.9033338591399158, + 0.906307372087264, + 0.9276637065009846, + 0.8689317693702764, + 0.8427719868983278, + 0.858611159693586, + 0.9082535731299035, + 0.8079662878574655, + 0.8627826207056297, + 0.8836204058420322, + 0.8850429046106282, + 0.881535813090887 + ], + "mean_band_energy": [ + 63.22727522132467, + 62.164163414456986, + 66.21133109915694, + 66.19830646178659, + 77.16519400786079, + 84.0704950133244, + 94.03421610024874, + 97.41452681199941, + 100.43272616139954, + 100.3193645087018, + 101.53045890882764, + 96.09535938275604, + 102.89386056299172, + 115.20716353296957, + 119.14145563461382, + 111.14814091866963, + 58.6415154233939, + 61.25507929140281, + 69.91896599667207, + 75.48906114133739, + 86.96838893558598, + 84.23276677565947, + 90.65629866168338, + 92.24638589495575, + 98.96458336823099, + 105.13117893028132, + 108.46240581594157, + 101.21893528919541, + 112.19452048347766, + 109.83113017543813, + 118.1201138711481, + 119.34985513610084 + ], + "dft_magnitude": [ + 2949.935222931593, + 24.562804623451264, + 318.9761705627184, + 26.10538695560225, + 184.66577238288508, + 46.66867466621953, + 134.7421778797087, + 8.954938950327563, + 77.02773989468865, + 14.637762934336816, + 55.81720437022133, + 19.716999036063164, + 74.85040528889081, + 9.409872328512437, + 55.93835444798383, + 9.02347488597358, + 12.808602426478046 + ] + }, + { + "layer": 3, + "mean_snr": 0.8950391029942858, + "per_head_snr": [ + 0.8781079709504299, + 0.8881969684743728, + 0.9397026237343719, + 0.8698036036129988, + 0.8050474617776899, + 0.8968112060145671, + 0.9045151093714487, + 0.807381318999935, + 0.8367308537805377, + 0.9327607523696343, + 0.9531329656682773, + 1.007743003342281, + 0.8734899698531386, + 0.9294143738928913, + 0.902748363071712 + ], + "mean_band_energy": [ + 57.727170916281146, + 57.67784704683533, + 73.14292728386846, + 88.46579111597799, + 80.74815052476053, + 89.01608345883417, + 83.07217591580836, + 91.80584431849165, + 104.30910893147339, + 110.44510151119363, + 123.78652037779484, + 109.01275173631142, + 118.05655090211381, + 121.34178637761994, + 120.57545863889912, + 124.63298453959867, + 59.60292842185643, + 68.38299503715321, + 78.02702885696438, + 78.14025688526236, + 87.06043606701864, + 84.1450668781682, + 86.46322991477638, + 95.22600590941633, + 101.70481050381483, + 112.43751358923015, + 126.0743839518271, + 104.249518057588, + 118.32283593759672, + 123.28337942821632, + 122.9571325990781, + 118.80668488350733 + ], + "dft_magnitude": [ + 3118.700460517337, + 15.556023842812634, + 382.6713315812345, + 12.990463071767396, + 134.53383310438238, + 23.444509688524168, + 185.69933983793265, + 16.477969007464996, + 96.93121203321063, + 5.172390330022904, + 53.96548310310001, + 30.341484206611916, + 99.30744992188468, + 29.697691105717915, + 81.58433875067001, + 16.694683861231276, + 35.438761029472516 + ] + }, + { + "layer": 4, + "mean_snr": 0.8523994834514139, + "per_head_snr": [ + 0.8216568746962811, + 0.8446074551524417, + 0.8030084443810156, + 0.9480683524516188, + 0.898337382563474, + 0.9214217296612018, + 0.8368271794942104, + 0.8512978653495116, + 0.8837215692693817, + 0.7792136268648919, + 0.6934645280630677, + 0.7192387864937253, + 0.9355578917277239, + 0.8879167355053139, + 0.9616538300973487 + ], + "mean_band_energy": [ + 51.93834703825382, + 52.20887783752365, + 55.988391756488, + 61.865886724164845, + 66.77310048062982, + 76.6313373966355, + 78.75766613967407, + 90.03591707765875, + 84.84827250589417, + 91.62478706921424, + 100.48684882102485, + 104.04841662887928, + 103.47754942113176, + 106.79870488595834, + 122.15505881036127, + 116.581428832852, + 51.29149381638669, + 53.923810868825186, + 56.03894948554441, + 59.10088059481812, + 64.98425441618816, + 81.5787236329148, + 84.74382458860266, + 89.88364836678139, + 87.50179952612307, + 91.11994152367946, + 108.7126141723765, + 107.93090186231647, + 94.94406898025845, + 124.20173813264027, + 128.43797694965093, + 123.22161373247164 + ], + "dft_magnitude": [ + 2771.8368320759228, + 36.028759686024344, + 412.50341245906714, + 13.726572308091187, + 234.14340149232493, + 16.70439742492716, + 137.1136814271575, + 30.24880382699429, + 149.49487836540334, + 19.14202524600788, + 72.42583413480138, + 24.59128677203252, + 59.8464385686936, + 31.11036489776278, + 54.22421684814323, + 19.2638478053262, + 89.6763982587454 + ] + }, + { + "layer": 5, + "mean_snr": 0.8983746804859686, + "per_head_snr": [ + 0.8193609892647016, + 0.9260867359548666, + 0.9137893852568458, + 0.918777332198396, + 0.9408273728359033, + 0.9510238906290004, + 0.8910762574793254, + 0.9806916434993093, + 0.9161654127710801, + 0.8991636424028484, + 0.753947330701634, + 0.8701247490451012, + 0.906811320722949, + 0.9783003943951278, + 0.8094737501324405 + ], + "mean_band_energy": [ + 53.841723537251724, + 53.68201852628804, + 53.73731367528797, + 65.42499491452472, + 68.51433538820318, + 70.11922951733092, + 73.69583818881692, + 77.32693831352665, + 84.12262053682697, + 89.96389321740978, + 85.03920674691781, + 89.34541600445662, + 97.84671223213705, + 88.87306663318877, + 98.49031558677977, + 99.60944746313817, + 49.38533807729341, + 55.46256608383531, + 57.80766900430297, + 65.03305503497485, + 60.916070937763564, + 65.30328816644713, + 75.01282949267359, + 78.39572732945349, + 83.54572834044353, + 88.68984241101595, + 84.97619038701262, + 92.87720430540931, + 101.53524701968439, + 90.13968251067854, + 95.52347671065148, + 95.7992425400889 + ], + "dft_magnitude": [ + 2490.036228833814, + 6.222358903583236, + 292.4938944624229, + 16.91882088115599, + 118.09049191870348, + 7.536383720485942, + 89.88811911043575, + 26.737412371049007, + 66.30111527626559, + 14.796767128018086, + 79.65747816321158, + 4.275488961810122, + 67.9775510900032, + 7.036032951542521, + 45.120197309245256, + 5.362965495852804, + 42.05499710972026 + ] + }, + { + "layer": 6, + "mean_snr": 0.8720639409832329, + "per_head_snr": [ + 0.7580074560720603, + 0.7813313433646574, + 0.8678770243566787, + 0.851516315886522, + 0.9204793016942228, + 0.9480297222829537, + 0.9187874072107661, + 0.9077281540346082, + 0.9072541757624251, + 0.8263469524305024, + 0.9094180066085535, + 0.8954877702637083, + 0.8835959679276898, + 0.7547993962561921, + 0.9503001205969539 + ], + "mean_band_energy": [ + 49.18515193966429, + 55.816713946363585, + 60.53086792553809, + 70.12453946112814, + 66.7826710743822, + 69.9907959759564, + 80.16282432740827, + 88.34835046393174, + 82.77869761254418, + 87.91634996690227, + 88.41283532575969, + 68.30971815986395, + 86.48927962537927, + 94.55761799992511, + 96.57587435963029, + 97.23278302325603, + 51.44352236767254, + 55.95799612215668, + 62.935043010762776, + 64.3825891403996, + 73.96741222361628, + 75.43993408294546, + 77.02820214680766, + 88.71896919684826, + 86.27867037657151, + 87.14833651823159, + 92.85033843995501, + 70.83815625407874, + 94.91668206184728, + 96.74193809898355, + 95.10014811347165, + 99.39767773252994 + ], + "dft_magnitude": [ + 2516.360687074512, + 21.33128497070681, + 214.37588324860894, + 11.00745149633776, + 151.11715565371432, + 14.28361661299911, + 134.51018407157284, + 6.835843979588583, + 66.17550999171418, + 23.17668059353286, + 32.74638385133509, + 13.514450398929792, + 107.52415430092148, + 17.32510660978111, + 66.28317392159997, + 14.384996911425596, + 25.484245212489895 + ] + }, + { + "layer": 7, + "mean_snr": 0.8789643180860965, + "per_head_snr": [ + 0.8807252911004928, + 0.8460707351182335, + 0.874843508342552, + 0.7754628961162433, + 0.7704099431887282, + 0.9141829748565781, + 0.9047707703509155, + 0.8930547651876752, + 0.9084019387889765, + 0.7996069797493979, + 0.8607129812643747, + 0.9786265300300429, + 1.1130305477193845, + 0.9796615007873196, + 0.6849034086905333 + ], + "mean_band_energy": [ + 37.02262682620489, + 35.47583564227504, + 41.31861197501293, + 42.9578987528302, + 47.231045971254254, + 52.47549966746489, + 59.0367603525212, + 61.20566526510922, + 66.79518516436362, + 72.86818306770587, + 74.25722645539066, + 71.32246893407677, + 72.84604706175904, + 83.59895574477235, + 82.80886634847636, + 79.86138557029335, + 37.52778263360855, + 37.4105491266149, + 42.10884797134267, + 43.63925007097383, + 46.69185500455284, + 47.75342307318609, + 59.146744423838996, + 61.50499711829331, + 64.31220916680259, + 76.58603958118186, + 72.98516174330221, + 68.8130142271687, + 80.92399579022181, + 82.71732024652142, + 86.09736465306017, + 77.41238878334518 + ], + "dft_magnitude": [ + 1966.7132064135258, + 3.8120444038760968, + 310.37469836412697, + 13.59407199217209, + 130.30024599974388, + 8.328520027849502, + 108.21173945958824, + 7.84706780839774, + 66.83152040005587, + 5.992098982297149, + 40.7850024157833, + 16.754591796424172, + 50.017602245953185, + 17.526120079639323, + 34.07253270016922, + 12.41473755048936, + 24.492543330100034 + ] + }, + { + "layer": 8, + "mean_snr": 0.8836535990080051, + "per_head_snr": [ + 0.8460334100104591, + 0.9522547236548813, + 0.9582309103770398, + 0.846800659994265, + 0.8428088542874685, + 0.8404268186295426, + 0.8872752527417873, + 0.9225963279379896, + 0.9207600782134212, + 0.785314588415719, + 0.9598559511764154, + 0.7941747023137655, + 0.9248024607710287, + 0.9182498734019604, + 0.855219373194334 + ], + "mean_band_energy": [ + 42.01059942768922, + 47.3731732590832, + 48.474779067840714, + 52.49157928372903, + 54.83807575029514, + 63.65739218689993, + 68.51654360667277, + 74.54521796273558, + 72.44456030266653, + 81.58059464182257, + 85.18425413465569, + 76.38249979517659, + 91.57297380950598, + 96.88993111927945, + 100.09890582775955, + 101.51827266304875, + 43.12631068314252, + 43.992098811624615, + 48.78112889651843, + 53.93456400599871, + 55.26408828728102, + 56.76496703956662, + 66.10995584486325, + 70.18493581081354, + 74.81584052665767, + 82.04277763622652, + 83.93616656691866, + 75.9609240624297, + 91.1618182732987, + 89.14449566608423, + 100.17680037056941, + 97.64023285302649 + ], + "dft_magnitude": [ + 2290.6164581738803, + 16.468687847285604, + 330.12852828181514, + 12.966848873872202, + 169.12532365225314, + 14.759171604206868, + 129.00217939682025, + 11.416070235338779, + 86.49407576720056, + 7.610420375382409, + 59.54683441234901, + 14.81423780394406, + 88.78387724206006, + 9.813678122729643, + 60.43554038934255, + 14.133858472932772, + 37.590855421210335 + ] + }, + { + "layer": 9, + "mean_snr": 0.8382011107690751, + "per_head_snr": [ + 0.8269818098839744, + 0.7658871561840301, + 0.7763407296557825, + 0.8002838969690358, + 0.9345295830501952, + 0.945048328393854, + 0.9045802774845985, + 0.9216581057082486, + 0.6925947480528367, + 0.7713962416255481, + 0.793568060051284, + 0.7710141902418264, + 0.8405613105763723, + 0.989061663372214, + 0.8395105602863269 + ], + "mean_band_energy": [ + 39.83799613357839, + 41.293951113045495, + 43.46734321959911, + 48.97874178493377, + 53.69862244113673, + 62.03487555237722, + 66.83802974059887, + 68.5272849198994, + 72.70807285207378, + 75.36330741041812, + 77.82608640500005, + 87.44555955469848, + 85.48554536032762, + 96.26439057385402, + 91.90903175553412, + 75.23635419151805, + 41.344680726448715, + 41.98846691607086, + 42.623255780067375, + 47.596231252235185, + 55.22003956549138, + 61.21994661916312, + 67.94398695978447, + 72.8559747592587, + 73.5134389478176, + 74.98909004365348, + 77.91364122440804, + 86.48746371884711, + 91.16411069110985, + 98.00943041776814, + 94.80935697589238, + 94.05264675115664 + ], + "dft_magnitude": [ + 2208.646954357766, + 27.781623072769747, + 347.1485405716594, + 20.14433463211495, + 189.62188350910492, + 17.787966215797802, + 78.87646986601236, + 13.87498742688324, + 58.625583485715254, + 22.526194371866854, + 45.20985491793307, + 25.0994381884484, + 28.407763120625933, + 19.370065531792804, + 43.66324327204968, + 14.937807682422642, + 56.04047680002918 + ] + }, + { + "layer": 10, + "mean_snr": 0.8753236688731931, + "per_head_snr": [ + 0.719324297800902, + 0.7840923688860933, + 0.7476003045190989, + 0.9792155051693658, + 0.9408222196209143, + 0.8404770280187481, + 0.9219030996107651, + 0.901222070417766, + 0.901662732471147, + 0.923702630327743, + 1.022415862879361, + 0.9109148548338776, + 0.8307672415053565, + 0.9001315918205821, + 0.8056032252161762 + ], + "mean_band_energy": [ + 42.0707267034265, + 42.198893949030904, + 49.46918346811408, + 52.43925753050805, + 58.15863543053413, + 69.38120075546004, + 66.11891088695897, + 86.9462961432735, + 83.23288414033988, + 95.27248577099469, + 103.45737186106007, + 80.34013438692334, + 93.01229369077922, + 98.62053270427947, + 98.45103136031497, + 99.36831624636606, + 43.6528114990881, + 45.02865878864777, + 54.178690157107184, + 51.24764615093963, + 58.09485998537405, + 64.05118654758824, + 65.02467233618623, + 86.63902940766347, + 85.06700031259285, + 94.8839603115869, + 103.00186471227262, + 81.09408874085487, + 88.91929591916366, + 99.97451883064471, + 95.00587623020529, + 96.41897085776024 + ], + "dft_magnitude": [ + 2430.8212858160396, + 13.553777792707452, + 389.5197371109384, + 14.044775014273752, + 127.17387161302, + 10.575574180419302, + 150.7405449963445, + 6.001216968844953, + 86.22773489031299, + 2.828240246536082, + 23.835465358420624, + 9.887275808026681, + 125.92054266467981, + 9.212820926538873, + 44.70726690972544, + 11.7738525118951, + 56.989068429003964 + ] + }, + { + "layer": 11, + "mean_snr": 0.863914637884421, + "per_head_snr": [ + 0.9466750184082036, + 0.874293890640863, + 0.9440971896618934, + 0.9484626839374708, + 0.865252703431502, + 0.8961185925300532, + 0.8184615671079147, + 0.8141099060775648, + 0.7648958436478436, + 1.018400535484538, + 0.9338051838193483, + 0.8601090234428129, + 0.7598115282125936, + 0.764482763818932, + 0.7497431380447807 + ], + "mean_band_energy": [ + 35.927152394552316, + 39.721639238459026, + 43.09346399597233, + 48.671575548514205, + 53.4837060871236, + 54.22184291680548, + 61.79163691032621, + 69.07324354224632, + 70.23818379132072, + 75.24296927751715, + 79.04578427228871, + 72.4184776109874, + 90.78544221418855, + 84.57112210255801, + 93.6970835099785, + 94.31205649407845, + 34.50003874773738, + 38.133683623290246, + 44.0208546976067, + 46.85591993316401, + 50.55279332263423, + 58.67520080760137, + 63.71637840372143, + 69.78079672448858, + 70.30035461251734, + 77.31837776378991, + 76.74593575479973, + 72.39414191596619, + 89.69428915166196, + 90.91647996480584, + 86.66891704038792, + 79.76988350797586 + ], + "dft_magnitude": [ + 2116.339425879066, + 14.382658127191558, + 328.63379400921156, + 16.323035658278215, + 167.66121766593088, + 23.799082315025416, + 112.03166376322999, + 17.573450246289983, + 63.475958278492406, + 29.718221593587007, + 58.085924341227305, + 19.08061965643192, + 88.88134003914878, + 10.015570971055082, + 51.86287400053742, + 11.404596004598233, + 27.815396065430377 + ] + }, + { + "layer": 12, + "mean_snr": 0.8602298890826269, + "per_head_snr": [ + 0.8089383993162428, + 0.9572392526457023, + 0.9664124751680745, + 0.7906988004981506, + 0.8024997250280509, + 0.8896180238748794, + 0.871889169087802, + 0.9668885688058344, + 0.9084874324027377, + 0.780570438613637, + 0.8015864471870545, + 0.7650484196997961, + 0.87774352528257, + 0.8520700871023751, + 0.8637575715264949 + ], + "mean_band_energy": [ + 38.31720152941082, + 47.407632025973086, + 44.8164662001114, + 51.83116611721638, + 55.19466211287251, + 64.02549647000666, + 68.62233142534303, + 78.3379520520519, + 82.45029757488678, + 88.91195921611865, + 94.14887993624342, + 82.4972797736748, + 90.6041459334766, + 97.67753289923284, + 94.43475191510122, + 95.20683850576918, + 39.8703669380702, + 44.52302950347882, + 48.673754993422975, + 53.67886083561894, + 60.16890941929062, + 60.50042920347703, + 74.46732385450915, + 78.29859499937074, + 83.5732010428247, + 90.55357843407904, + 94.7021506497633, + 80.65714484135994, + 89.55706095445242, + 90.08399518007793, + 96.69800196283524, + 93.79874942720947 + ], + "dft_magnitude": [ + 2354.28974592733, + 15.365499074036752, + 365.7376335084897, + 9.29685157889329, + 137.66566876335003, + 10.666906343490112, + 129.34744096287707, + 3.3888995933423947, + 82.70596887495327, + 10.38570152110923, + 39.60835853192107, + 13.381167969101128, + 83.02399326193786, + 16.586265176613463, + 72.9804089895997, + 19.5231868821175, + 41.690733042100874 + ] + }, + { + "layer": 13, + "mean_snr": 0.8683865442694285, + "per_head_snr": [ + 0.8627381967421259, + 0.8192731132590692, + 0.834946810545642, + 0.9154532343065319, + 0.8757248894679983, + 0.9115045833141405, + 0.9430093091229955, + 0.8663733947888261, + 0.8988853317885692, + 0.7801366276717738, + 0.8476387211753635, + 0.7416289667576653, + 0.8508016409765593, + 0.954199261019271, + 0.9234840831048973 + ], + "mean_band_energy": [ + 49.0629162303804, + 46.5621842340857, + 57.0616409308144, + 68.29059391746004, + 73.86659192985732, + 81.13660612941435, + 79.9336849690278, + 81.10350528595694, + 89.09698350151078, + 87.66793569786833, + 94.40637514968464, + 80.66224309148835, + 82.89250915283907, + 87.6255209950799, + 92.43695236353784, + 82.16595972370851, + 49.44963034838532, + 48.06885608136396, + 62.73139298189383, + 71.56502246554534, + 76.00425511230569, + 79.14790324794879, + 87.21454406211159, + 90.67712579115366, + 89.76333496916293, + 86.32044637209543, + 93.11651301935939, + 84.82089017693234, + 82.42168724762574, + 86.57328846291541, + 92.33397342005101, + 82.6780276844372 + ], + "dft_magnitude": [ + 2496.8590947460016, + 25.321811455864413, + 244.14466552513548, + 7.226500136798584, + 129.27943137389246, + 10.897888592017727, + 117.69901416869347, + 25.037921165392735, + 77.17508089226777, + 13.144793869861127, + 23.72998581038766, + 2.216746877813934, + 27.549299851304863, + 9.619791804506603, + 33.61780499574805, + 7.813850215929731, + 6.726876031093525 + ] + }, + { + "layer": 14, + "mean_snr": 0.8605897605649192, + "per_head_snr": [ + 0.8912122066809003, + 0.9000551770634182, + 0.8998544853269544, + 0.8488944028116311, + 0.9004843341818903, + 0.838235851186814, + 0.7783192423634403, + 0.7704623696899922, + 0.7956301176602872, + 0.7902682341612304, + 0.8372527986770084, + 0.916115794887013, + 0.8856649154887829, + 0.9129359069913608, + 0.9434605713030644 + ], + "mean_band_energy": [ + 35.69602835205507, + 37.04348259815206, + 45.16688268493011, + 47.73472130351602, + 56.782981001169176, + 64.33234808576812, + 68.45540704564642, + 75.43439959715049, + 75.94789216099076, + 83.31197134649058, + 86.35955788663644, + 86.10214901967707, + 85.7125977435232, + 95.88165781806497, + 85.56722906392446, + 88.27695150748656, + 36.12003466038903, + 38.96985633635849, + 47.449453127033614, + 47.6283871542254, + 59.420692932008706, + 64.3691876074084, + 69.15271368075804, + 72.01881736497914, + 77.34846511272971, + 84.68293236988248, + 84.44301243245893, + 86.82429119700328, + 88.5636895842322, + 87.85520199591974, + 87.61567433125178, + 78.21479667500928 + ], + "dft_magnitude": [ + 2228.48346377683, + 18.593562234922604, + 356.81553035618174, + 10.666221919318508, + 151.76494265441616, + 8.56473472377499, + 94.45244220106255, + 5.267928064657876, + 64.03929689927831, + 5.291402058954698, + 53.15698437316077, + 14.671617530515435, + 59.46153276780088, + 22.18175231694207, + 44.36609475281995, + 20.44896646771102, + 48.878840177354505 + ] + }, + { + "layer": 15, + "mean_snr": 0.8399989334850277, + "per_head_snr": [ + 0.9214821624076259, + 1.0048290595895124, + 0.7834763447204613, + 0.7829137290410554, + 0.8517864118457287, + 0.8127411325782724, + 0.9263063686584299, + 0.7418661241914177, + 0.8361003338899884, + 0.8446476079585562, + 0.8039134085942787, + 0.8364640489112409, + 0.853081421097775, + 0.7690495493914961, + 0.8313262993995784 + ], + "mean_band_energy": [ + 34.987498976631066, + 36.59377978914374, + 44.767428614915026, + 49.42202277829577, + 54.11843715031808, + 58.678205371713, + 67.59934441024204, + 75.69735211927933, + 80.28285835632347, + 85.20347713881526, + 92.70704606634305, + 86.76461390636038, + 88.2572862489169, + 95.54158316819525, + 94.78560340718835, + 93.33906484587827, + 35.66555931393108, + 38.08775443210843, + 43.09478261038307, + 49.75833557618266, + 56.22540259751413, + 60.58370507196672, + 71.25839398865168, + 75.104461624725, + 83.06315019282123, + 85.29967173391587, + 92.29806191285273, + 89.02765748907865, + 92.37929364030539, + 99.03340749122223, + 90.10374258854281, + 87.84685764695396 + ], + "dft_magnitude": [ + 2287.575840259715, + 14.212304868500492, + 407.131267673473, + 7.192583606011308, + 154.2063914112457, + 19.179124015159726, + 120.23598330950185, + 11.416641251890326, + 86.19563881429814, + 8.791072306709916, + 42.74216418051086, + 2.4218305018833877, + 61.80079128401421, + 11.258881574361121, + 70.0074696730642, + 3.443633350444009, + 44.388060107954516 + ] + }, + { + "layer": 16, + "mean_snr": 0.8587557853673019, + "per_head_snr": [ + 0.9052854564452726, + 0.844147986333058, + 0.8043629862271556, + 0.8440212221025841, + 0.8045349145054771, + 0.8538170691002652, + 0.9033369615929995, + 1.0396665382407673, + 0.9118556177639298, + 0.8033455360393694, + 0.8405422733334614, + 0.8060908557305604, + 0.8072584715127525, + 0.8654576949611582, + 0.847613196620717 + ], + "mean_band_energy": [ + 26.705517263600118, + 33.89726086207065, + 38.05578453340883, + 49.83661666656591, + 54.20932229995073, + 64.2070018773208, + 70.32683991799837, + 74.96463414181889, + 77.2988454100343, + 87.90749402484025, + 91.22963711565826, + 91.42300505652715, + 88.78892905415302, + 96.10508400567075, + 93.72208687359267, + 95.48807944371428, + 28.987572582886642, + 33.94489829236541, + 39.200260398297345, + 46.10902424337167, + 52.820462543577655, + 63.392769348710026, + 68.44474085768933, + 74.3316429181783, + 79.75164898930998, + 89.08162880652159, + 90.1919229047527, + 89.51954442827186, + 88.32021341716327, + 95.83718644300066, + 88.69052719379216, + 73.74078717211887 + ], + "dft_magnitude": [ + 2236.5309690869326, + 28.106361610069023, + 431.2757624537967, + 33.96272159498535, + 173.04327047178035, + 20.20197878498152, + 121.40708599473065, + 25.41746467374297, + 88.59483349588506, + 19.025184994903167, + 51.55508988156844, + 22.407178629849977, + 51.91754146295007, + 16.58244508256347, + 55.32217811462666, + 12.812506430885875, + 83.04234637520153 + ] + }, + { + "layer": 17, + "mean_snr": 0.7966276189214809, + "per_head_snr": [ + 0.8807590006236313, + 0.8919127281959477, + 0.80026968083249, + 0.8316670236749734, + 0.7680019342543857, + 0.7150089438716694, + 0.6668872730145319, + 0.6590243946221687, + 0.6944441462771951, + 1.0017752739227888, + 0.8326305405899661, + 0.9048717639403626, + 0.7698598122437253, + 0.7670351994753573, + 0.765266568283021 + ], + "mean_band_energy": [ + 17.140166525696817, + 21.948925898329797, + 26.00944684649178, + 31.878713716198735, + 33.25566740135414, + 39.63833442955944, + 40.37890674050016, + 46.348401314494126, + 57.42319646217829, + 63.86766282332557, + 77.10073633799553, + 100.51932733776474, + 111.83679107283994, + 114.34665430370528, + 98.10492661442471, + 79.18514627017517, + 18.168184495582544, + 20.845567321175558, + 25.99769138370019, + 27.9212960366741, + 33.23335822307548, + 37.91196481860148, + 39.97564282661654, + 45.451317607671314, + 56.124330819773746, + 66.61219410137227, + 76.5390406846506, + 100.68508839088709, + 106.50180426697666, + 113.34094613769527, + 105.83944495892325, + 94.91304305908264 + ], + "dft_magnitude": [ + 1929.0439192274928, + 23.161154818353378, + 648.704689683213, + 16.669886546479596, + 285.95485716702507, + 25.561384374259653, + 116.57946843270877, + 27.290191895689617, + 74.20950612037514, + 15.31972342085301, + 91.89241901761146, + 10.90221957158234, + 49.55173550831749, + 5.161500085802501, + 65.36921254134462, + 18.76746172535497, + 81.78524790593224 + ] + }, + { + "layer": 18, + "mean_snr": 0.78775718505282, + "per_head_snr": [ + 0.7787020882608612, + 0.8836621143125045, + 0.7708588064062476, + 0.7376072540523577, + 0.7571834673683847, + 1.0138515560803838, + 0.9861827991772208, + 0.849962203564633, + 0.7798933910905023, + 0.7066257162751138, + 0.6789645300243288, + 0.7186035200490416, + 0.7052081877783505, + 0.7379411892878411, + 0.7111109520645265 + ], + "mean_band_energy": [ + 16.8219949124682, + 18.814688335574985, + 22.485544685888993, + 24.875121203072123, + 31.70223456148984, + 34.29157581736792, + 41.086451731259864, + 47.89833978071608, + 53.58604517651663, + 64.05284020008372, + 68.3037387435925, + 87.09000534923689, + 104.45496770333025, + 99.8804841688664, + 102.27972250856978, + 68.3215531291934, + 18.292552347140827, + 18.982212440056554, + 22.738848804975436, + 26.60028960472689, + 31.226579700317462, + 35.0437065214853, + 41.79115604325713, + 46.56324476369977, + 52.96784326793371, + 61.80662584692538, + 70.5601523960848, + 86.06243084441196, + 105.68193459200509, + 100.93013299363616, + 103.97039426060967, + 79.63657204597317 + ], + "dft_magnitude": [ + 1788.7999844804667, + 11.38992450012574, + 593.9371994933247, + 16.03072336359017, + 273.6768486814681, + 8.821385211358526, + 121.56535087154049, + 13.068899189574847, + 67.27092181306827, + 7.242341801745515, + 75.43872587005988, + 11.632143758248336, + 70.78971040844546, + 18.320771317317362, + 25.554363501822284, + 8.260827544782904, + 12.899661609586587 + ] + }, + { + "layer": 19, + "mean_snr": 0.8522195785194866, + "per_head_snr": [ + 0.8442515629771905, + 0.893840382274619, + 0.8700488121785129, + 0.8726765685281045, + 0.839593025574465, + 0.8894922880690825, + 0.8698728412508409, + 0.900529077637892, + 0.895724456833772, + 0.8432614298887913, + 0.8083838504343176, + 0.8362522329916566, + 0.8480925683988522, + 0.7696175845154092, + 0.8016569962387925 + ], + "mean_band_energy": [ + 30.95104202709308, + 34.80547424631327, + 45.24414446488866, + 47.31289678158693, + 57.22919320850351, + 64.59257871270057, + 70.59870573887432, + 76.56923273211437, + 79.41215859841908, + 83.26823097925869, + 86.97956094771344, + 84.74906209901953, + 86.0590429526949, + 87.76924872206457, + 81.97169216561718, + 86.30207939682404, + 31.110023967005326, + 37.216679172432016, + 45.88390802979719, + 53.28702138117033, + 57.62030675972786, + 66.73315455953843, + 73.20801916970467, + 76.69547358528125, + 79.70390381872775, + 80.77445410735193, + 84.46927496207324, + 83.25242753995408, + 86.0986912079078, + 86.74706012098581, + 84.30921389278842, + 68.63664298676547 + ], + "dft_magnitude": [ + 2199.5605990348977, + 28.838717911284515, + 353.6331822142537, + 4.983217962452469, + 152.27121382536887, + 16.60490996960964, + 90.70395855718135, + 17.631490152279216, + 73.31819658001707, + 18.899163112741828, + 52.575569563821084, + 16.472956831100916, + 53.56846834905191, + 16.33677121556528, + 45.93233255795692, + 27.510913736306673, + 37.86283521182486 + ] + }, + { + "layer": 20, + "mean_snr": 0.7725419482918956, + "per_head_snr": [ + 0.724658490063252, + 0.743321021471559, + 0.768977714108209, + 0.8131121722033728, + 0.7665078348512928, + 0.7519241317624167, + 0.7664635164836522, + 0.8552641562899387, + 0.7898747500681003, + 0.7823846654732366, + 0.7414146620798708, + 0.7784150947879851, + 0.7328509943770007, + 0.7399213918950315, + 0.8330386284635158 + ], + "mean_band_energy": [ + 14.105502050971264, + 15.269417290814609, + 17.26300412961691, + 21.494895069851985, + 27.11378561089342, + 30.333220428972652, + 35.84957622158945, + 43.20527066708, + 51.73493778774529, + 60.91514368406336, + 76.61154501627227, + 90.32509208614789, + 97.39930116466333, + 99.86788574000651, + 86.57236614101417, + 83.02468137788999, + 13.287276897723647, + 15.207385337323421, + 16.774593041016274, + 20.862985764285913, + 25.7968010903049, + 30.908255864252002, + 36.628941177109844, + 43.698408511193186, + 50.431820048866726, + 60.289440821338275, + 71.44538622880245, + 91.70532257176568, + 96.13020109236975, + 98.07812466969159, + 96.283848490789, + 77.27108013598988 + ], + "dft_magnitude": [ + 1695.8854962104156, + 4.935691444144025, + 624.6266364595255, + 7.148989569165758, + 242.07556814542443, + 11.914747649794968, + 99.37336122855677, + 4.9964905090452545, + 86.37349283494208, + 9.1977182544234, + 73.75052886226355, + 18.938537828990295, + 60.046820819781885, + 20.961489303348795, + 63.08046688970716, + 14.563171427496526, + 69.02772383091826 + ] + }, + { + "layer": 21, + "mean_snr": 0.816945452358716, + "per_head_snr": [ + 0.7221815816693008, + 0.7195213284334091, + 0.8369345109744806, + 0.8298742186695082, + 1.094906730539877, + 0.87027382366238, + 0.8249168385908908, + 0.7157748407755365, + 0.8115819076226518, + 0.7980876369150913, + 0.8038402038217839, + 0.8234916516989926, + 0.8034987247362494, + 0.8087863306828526, + 0.7905114565877358 + ], + "mean_band_energy": [ + 20.018343809372517, + 23.096541936276687, + 32.4007896035525, + 35.99024215898565, + 42.62922459530467, + 45.569715944719945, + 52.93590037935722, + 56.69633497754884, + 60.17388366931559, + 66.59886364329401, + 70.73965577436694, + 74.85058810587172, + 85.41775613827106, + 78.62570657057276, + 78.70523814418857, + 64.76980468207518, + 21.700765704865844, + 26.675529032920505, + 32.44068070020119, + 35.22366366666486, + 40.25089839730034, + 46.40969414849887, + 49.730739337321744, + 56.46190198424733, + 62.0496649493649, + 70.18556494509214, + 73.14484810312543, + 78.29338699599154, + 85.88561298772068, + 80.61984454138589, + 71.67555236052294, + 76.45891262095496 + ], + "dft_magnitude": [ + 1796.4258506092528, + 11.331889331529302, + 395.0081479272025, + 10.188114025341505, + 159.8370071513595, + 12.348713156090733, + 84.14921500280052, + 12.163503246766645, + 59.85884094760309, + 20.589678372387198, + 68.85263348561107, + 13.582613700980767, + 69.53589481356089, + 21.5591383864169, + 42.7619281768879, + 21.251491933109808, + 36.626741300948765 + ] + }, + { + "layer": 22, + "mean_snr": 0.8430227494564317, + "per_head_snr": [ + 0.7156835265591137, + 0.7805193129716316, + 1.0452430323072943, + 0.8329320318005022, + 0.9851541231134793, + 0.7258702776191094, + 0.9445673944932297, + 0.8734372234311689, + 0.7426092363969491, + 0.7929314081619467, + 0.9120305880889149, + 0.9249239935983142, + 0.7700979712019524, + 0.7631013724824184, + 0.8362397496204503 + ], + "mean_band_energy": [ + 21.28970578604054, + 26.074512482000205, + 31.574118335222472, + 36.181341138313414, + 38.60215607738168, + 43.2488003579048, + 49.21729443510251, + 54.53029823285302, + 59.306739756073966, + 62.08370089423183, + 63.86738597930468, + 72.6847747058541, + 77.80452705180745, + 70.70642765870723, + 69.81204675963727, + 70.63157732370306, + 21.79494291074618, + 28.142387534241756, + 29.724040123201267, + 35.80951030886611, + 38.800930797439946, + 44.28287091165313, + 48.888616029754026, + 52.54651158494953, + 60.09068274364613, + 61.1805059760361, + 67.96194043555528, + 74.71993515253054, + 76.9889961225422, + 71.95894589434195, + 69.84495339424265, + 60.472630828328555 + ], + "dft_magnitude": [ + 1690.8238077222136, + 7.864542333573953, + 345.3285618388297, + 16.010922560843493, + 133.57771472840258, + 13.587306268470256, + 66.22620947369258, + 9.080679021831438, + 61.65339668052489, + 8.908644373115374, + 66.73849735504312, + 14.286055200155584, + 46.104201922468064, + 9.65197520006132, + 47.45275706193307, + 12.89500482708736, + 39.685654246817194 + ] + }, + { + "layer": 23, + "mean_snr": 0.7985369553173598, + "per_head_snr": [ + 0.7886320866234813, + 0.7575422865883483, + 0.7889303323285157, + 0.8960196649314865, + 0.7717971889313467, + 0.8479685129709412, + 0.8140005512577724, + 0.746282722869754, + 0.7787312513353729, + 0.775133467334164, + 0.8414900919384453, + 0.8314673866779129, + 0.7292719138638631, + 0.8537976883243008, + 0.7569891837846918 + ], + "mean_band_energy": [ + 16.56939840336626, + 18.556857226393117, + 22.75551803926696, + 24.907570680832617, + 30.067195353016984, + 32.71537722336196, + 39.48236865437494, + 45.04747290477607, + 50.9834934672097, + 59.40557108198748, + 70.1345728482251, + 72.07140794419998, + 79.47821885370823, + 76.2158212715648, + 73.3246999125287, + 73.03753985061545, + 17.28041372075055, + 18.70892057878494, + 20.723017258381738, + 26.320843894235562, + 29.386306370973035, + 32.8162356148801, + 37.78264254517868, + 43.6375088880968, + 52.17007515368331, + 60.71448893293071, + 69.23263499141896, + 71.81584091150407, + 80.806804695302, + 78.91446686561603, + 75.26706472777808, + 64.36137877718308 + ], + "dft_magnitude": [ + 1564.691727642126, + 2.6008522810777395, + 453.7892025522052, + 10.110700415510355, + 152.21926712097508, + 8.256319782668632, + 93.03825953783621, + 18.133547234910893, + 67.54247037788134, + 8.05016516809479, + 54.0555163809192, + 7.901849001057051, + 64.38310808721312, + 7.450924458389801, + 53.960074458674065, + 12.003793633735901, + 33.80287765179958 + ] + }, + { + "layer": 24, + "mean_snr": 0.795298813031923, + "per_head_snr": [ + 0.7504778094384259, + 0.7328410345180303, + 0.7815947511303392, + 0.8693858695833739, + 0.7805220431262043, + 0.8492382878281985, + 0.8199500257405815, + 0.791236836343838, + 0.8009241214861074, + 0.8171034890984674, + 0.7475733469790864, + 0.7516026909935744, + 0.7996395042308808, + 0.8118633662675878, + 0.8255290187141502 + ], + "mean_band_energy": [ + 15.168599784507169, + 15.694309180199479, + 16.976198768175053, + 20.481355641807234, + 22.8245473187775, + 27.813585996836796, + 29.416394058221425, + 38.78882618406748, + 44.160576128384584, + 54.28541925830533, + 62.91377012868097, + 81.02785316293833, + 85.12061203311296, + 91.79175151922546, + 83.66683318243093, + 71.03995857987319, + 13.864807186896547, + 14.905033722659072, + 17.10592496023924, + 19.640653319973573, + 22.323509918217052, + 28.106301622800355, + 30.993606343720785, + 37.674688008723514, + 43.31998597936497, + 54.028467202123316, + 60.737754962232444, + 82.43077281446628, + 84.35288942523039, + 89.52917090866866, + 78.16121991469696, + 66.0823401564843 + ], + "dft_magnitude": [ + 1504.4277173720413, + 12.291956442137572, + 546.0178582823196, + 9.756610503569027, + 213.75470265573898, + 12.490977388817631, + 86.86458398416768, + 8.276534144239648, + 63.7729534486917, + 7.7601111890596375, + 62.373315830537884, + 3.1324165968211255, + 37.65096597675437, + 7.556954696155206, + 42.17663511376337, + 1.8821685295206696, + 82.21325718626326 + ] + }, + { + "layer": 25, + "mean_snr": 0.8405236742316415, + "per_head_snr": [ + 0.824456988850999, + 0.9866604250033754, + 0.7557914721410779, + 0.9635747318812682, + 0.8289683489034919, + 0.8435458580582411, + 0.7301511841174266, + 0.738754782874768, + 0.8184412875531154, + 0.8452388716250239, + 0.8568029724939256, + 0.76517309372923, + 0.7081281811264273, + 0.929050909362991, + 1.0131160057532609 + ], + "mean_band_energy": [ + 23.42302611402137, + 26.415792788847494, + 28.144657455148717, + 32.55876953191076, + 35.92817826610303, + 39.791939276937725, + 42.13610165574184, + 46.60591818690021, + 45.614255797347916, + 51.11257097585109, + 53.22205447322393, + 59.892969394199234, + 59.81715688280966, + 61.37654118748777, + 51.60832242562738, + 51.43735287371598, + 22.072182955206994, + 25.937310175733536, + 29.814025842556056, + 33.418967684120354, + 36.98299192043334, + 39.889276337231486, + 44.1522119225001, + 46.73508266785287, + 46.29862939363515, + 52.63155825703549, + 54.1714418667129, + 57.98792724611514, + 59.0976870996476, + 61.932719131780445, + 57.59244656021776, + 55.56259010855967 + ], + "dft_magnitude": [ + 1433.362656455213, + 10.845141913985849, + 234.26867531128568, + 12.573413945583837, + 106.68474410142169, + 12.890305706295171, + 44.840452581614116, + 7.9687171253574185, + 40.368529357934435, + 3.9763147226323374, + 27.8135435448564, + 3.6724385832571342, + 36.03799282934941, + 3.8001808152952194, + 31.505595088445702, + 4.984784400304747, + 53.211915193345476 + ] + }, + { + "layer": 26, + "mean_snr": 0.851962817895055, + "per_head_snr": [ + 0.755633573422343, + 0.7579667045741367, + 0.7686622339045773, + 0.9560888873643137, + 0.8979289576923534, + 0.9437531847400273, + 0.7900397088263958, + 0.8638682478491871, + 0.7715922813295265, + 0.86761798343569, + 0.788313658215343, + 0.8756770991156233, + 0.9441017670912457, + 0.9021536628797161, + 0.8960443179853459 + ], + "mean_band_energy": [ + 23.3066152063336, + 27.423740225060808, + 28.57323690029036, + 32.75751464599774, + 33.79222432573795, + 34.7322760485205, + 40.533787027587444, + 43.27673276825626, + 47.232459755374734, + 49.438108297460744, + 54.66956890100992, + 55.1945495879314, + 57.19719883954678, + 52.672292360384745, + 49.57837798397862, + 47.22068613116814, + 24.225950830793863, + 26.279524115603067, + 27.78800257640758, + 33.3027922302355, + 34.1941306303258, + 36.93449484503176, + 40.36230843245335, + 44.0092335204004, + 46.05914593473288, + 45.64308979698591, + 55.18299109246178, + 51.3376782302398, + 56.06184830499933, + 54.88771284749661, + 58.22630122927717, + 48.47550241011675 + ], + "dft_magnitude": [ + 1360.5700760322013, + 6.571809233187617, + 214.76461219079098, + 18.42511839245891, + 74.96017527105909, + 10.696410220608547, + 45.314488116090025, + 12.299245430714608, + 42.87817482228132, + 12.456503691771987, + 19.289162823838872, + 6.410169571198539, + 27.00757850193498, + 7.5208757049292005, + 34.68355179336566, + 8.706158081992712, + 6.6017800895790515 + ] + }, + { + "layer": 27, + "mean_snr": 0.8074989530415763, + "per_head_snr": [ + 0.8592476047698007, + 0.7508181642705052, + 0.7197090801026522, + 0.7893387446749992, + 0.8511251784218101, + 0.7290877871308126, + 0.711935180161036, + 0.6939547029385489, + 0.7868293674381285, + 0.8283095224036094, + 0.8099552840654278, + 0.8078524830763727, + 1.0377126390805718, + 0.9124527025264002, + 0.8241558545629691 + ], + "mean_band_energy": [ + 18.561651342805856, + 18.534479072961382, + 20.754143319962136, + 21.415897929996337, + 21.899344667996168, + 26.829301047913564, + 28.79579234425461, + 32.638859169652065, + 39.34536533674897, + 42.84999958956303, + 50.88005129113458, + 63.103638768083144, + 65.57807244935407, + 60.02327159502949, + 50.10588787956652, + 48.247131984421245, + 16.901826529870515, + 18.599308594518018, + 18.877788498363984, + 20.024402592331587, + 22.86925889819494, + 27.138054831102576, + 29.45076894997049, + 32.073772216466885, + 38.25091686294456, + 44.77219098964086, + 52.49377148747575, + 61.495285122589756, + 67.7519018485874, + 62.84810879704346, + 52.68410559542287, + 51.81107072084498 + ], + "dft_magnitude": [ + 1227.6054203248118, + 14.345608816409978, + 346.7086555264514, + 6.643901146244637, + 116.07019216543544, + 1.858923365178818, + 23.061203529136915, + 8.098911924168236, + 31.930127529594362, + 3.724866991948804, + 48.854427830448635, + 8.875964620343415, + 32.656861985774015, + 1.4269560972290125, + 37.17948703761309, + 1.8509914642714511, + 37.2041257195051 + ] + }, + { + "layer": 28, + "mean_snr": 0.8170522654932652, + "per_head_snr": [ + 0.7713821384325602, + 0.821911490333378, + 0.783066202312132, + 0.7923039076152967, + 0.7322759697619701, + 0.7527802386134902, + 1.0040270699758, + 0.798684896136075, + 0.8053917994134975, + 0.8862398729702957, + 0.8332523618312719, + 0.8291209717599353, + 0.7312053210381811, + 0.9676244465936442, + 0.7465172956114497 + ], + "mean_band_energy": [ + 16.36186242059949, + 16.709360953317248, + 18.54010874419915, + 21.558864283949045, + 24.202480541316003, + 26.097649394455235, + 29.839191015803454, + 33.15064049784015, + 38.44076939952649, + 43.94087617076921, + 50.90557629852928, + 56.90592820758629, + 61.66561097723973, + 57.03816010081609, + 55.55568420264019, + 48.21605967267231, + 15.878001992780742, + 17.880062244519333, + 20.498101671970765, + 20.792917912584823, + 23.473836868952585, + 29.07271784703976, + 29.445968571816124, + 34.200724058835945, + 35.81264822456972, + 41.78593131595997, + 48.89053811025063, + 55.58758471803427, + 62.163194378911975, + 61.730966145582684, + 52.01563862489922, + 52.79231727863835 + ], + "dft_magnitude": [ + 1201.1499728466063, + 2.0906398312115546, + 316.977732402202, + 12.250917488891407, + 120.07550331800009, + 6.748609715483844, + 43.993475745089704, + 5.490699280350495, + 40.061600622810985, + 10.465017864000497, + 41.78851340677116, + 9.564131293712597, + 41.492459007170744, + 10.81565961813931, + 32.29615913337173, + 13.036332638145911, + 33.771548758595145 + ] + }, + { + "layer": 29, + "mean_snr": 0.8479653375383622, + "per_head_snr": [ + 0.9231014205934811, + 0.9475225510960202, + 0.9412345939069413, + 0.7582717330524809, + 0.7509820535070348, + 0.770938090323196, + 0.7464858184195247, + 0.7584335685601529, + 0.7701735513140333, + 0.9113791757599456, + 0.9436999345069741, + 0.7603662178799525, + 0.8130438407273238, + 1.0198299581636305, + 0.9040175552647425 + ], + "mean_band_energy": [ + 18.780575574151538, + 20.845223797505817, + 22.366922834264166, + 25.085851043625016, + 26.958536204774223, + 27.908879485044235, + 28.695808208203108, + 30.56425540323637, + 36.02546757077559, + 38.832445958754185, + 43.1987499701613, + 49.17994279737103, + 50.99527113831502, + 53.44611826249097, + 51.36184701817453, + 43.650303342170595, + 18.96577290925127, + 20.066712681029692, + 21.996105070238297, + 23.626611732174297, + 26.065127774836114, + 28.102142751237547, + 30.55108927919758, + 34.218825349479744, + 35.570553698138006, + 38.19822288331511, + 41.84806382588427, + 50.66375851831538, + 50.69639480957015, + 53.305872083726264, + 44.907878597155744, + 43.49269623347145 + ], + "dft_magnitude": [ + 1130.1720268060385, + 5.028285954356013, + 225.56464678588938, + 11.24752862067198, + 95.9138399589748, + 9.036050849065973, + 41.982792739822166, + 11.750318733733199, + 28.751005446089486, + 2.897446758785734, + 27.775046410306047, + 5.888387102807895, + 17.93111513361236, + 2.940487538161339, + 26.86698380979104, + 10.605787865805993, + 32.20369783985677 + ] + }, + { + "layer": 30, + "mean_snr": 0.8960867507074718, + "per_head_snr": [ + 0.7330793399077485, + 0.8178542346310308, + 0.7384977888879741, + 0.9727062528047018, + 0.9202956831469776, + 0.9761754485616241, + 0.9331495744508655, + 0.861209307229864, + 0.9648338644351773, + 1.033451261162912, + 0.9435162934227872, + 0.9078942052049951, + 0.8852730095244224, + 0.8790354057503816, + 0.874329591490616 + ], + "mean_band_energy": [ + 20.60404890555557, + 22.6508256834245, + 24.613917557835126, + 26.251365595175255, + 28.149894879119117, + 28.66987309559203, + 31.56418833563807, + 32.474295094251936, + 33.076662017362885, + 35.14280653732849, + 37.4242294002912, + 37.248216472279, + 41.00547908170811, + 39.85541433728372, + 42.75211604031094, + 43.87412322350956, + 20.23017480936766, + 23.211881804279077, + 24.646522212923596, + 26.018819673978797, + 27.136512727830304, + 28.821429439738782, + 30.49034384911832, + 33.075196741184236, + 33.97377904952013, + 36.00843351383657, + 38.03770613431161, + 38.27326404106297, + 39.794905808012, + 40.69849058850535, + 39.97664461844254, + 40.43376759491639 + ], + "dft_magnitude": [ + 1046.185328863694, + 4.2076496593818185, + 123.72786097828852, + 9.531835161422027, + 57.51896619520991, + 2.515788180013646, + 39.28463681182892, + 3.9725240115567573, + 34.092509591409765, + 3.7764879992955884, + 25.191127389777144, + 6.621449211490879, + 30.750428907564224, + 3.7291931993304677, + 24.039481753379437, + 4.413859487952234, + 19.231078008999475 + ] + }, + { + "layer": 31, + "mean_snr": 0.9165189463594561, + "per_head_snr": [ + 0.8179093323112367, + 0.8862828479609517, + 0.9226479966436202, + 0.9085718354152299, + 0.8814179238677692, + 0.9599390931386885, + 0.9587119795613274, + 0.9278779253827055, + 0.9033555523544902, + 0.9526684119478149, + 0.9617333938511538, + 0.9844639754423178, + 0.8157672553118224, + 0.9612295510574488, + 0.9052071211452661 + ], + "mean_band_energy": [ + 23.5125351163419, + 26.982064797972466, + 28.20948354041848, + 31.64751048338091, + 30.854604911253986, + 32.74364097551732, + 33.44606046208225, + 33.96978882531813, + 35.3284308730756, + 38.23567777456624, + 39.52861048613798, + 44.87402845370132, + 43.878266156925356, + 44.74170726360597, + 43.791120902730064, + 42.744334951736725, + 22.839955602807766, + 26.392160570577783, + 28.167840826941855, + 31.4925665869093, + 31.468614495012623, + 31.674339750405654, + 33.048166427244254, + 33.71319967739903, + 34.050667393607185, + 37.29093219235195, + 40.81781217631251, + 41.582859424846994, + 42.90252600487511, + 43.772852910633965, + 44.13543015763345, + 41.50427930873255 + ], + "dft_magnitude": [ + 1139.3420694810566, + 6.726419938872668, + 118.94559196947299, + 2.757145233603332, + 60.776364088217235, + 2.7533789705858838, + 34.91543539578996, + 2.3718148948243862, + 32.864267399773354, + 5.554626376411899, + 20.91001910143291, + 4.793105072760294, + 18.05512019162352, + 3.4366147256911344, + 20.063302424322515, + 6.11578399596125, + 27.381818414255918 + ] + } + ], + "elapsed_s": 0.19134259223937988, + "loader": "raw_hf" +} \ No newline at end of file diff --git a/data/exp_wqk_spectral/gqa_comparison.json b/data/exp_wqk_spectral/gqa_comparison.json new file mode 100644 index 0000000000000000000000000000000000000000..e3e28d64a598b73513c2491aab22ed964664a9b1 --- /dev/null +++ b/data/exp_wqk_spectral/gqa_comparison.json @@ -0,0 +1,202 @@ +{ + "results": [ + { + "model": "Qwen/Qwen2.5-0.5B", + "short": "Qwen2.5-0.5B", + "theta": 1000000, + "T_train": 8192, + "n_heads": 14, + "n_kv": 2, + "d_head": 64, + "n_layers": 24, + "k_dead": 18, + "n_pairs": 32, + "gamma_rand": 0.919, + "gamma_text": 1.028, + "snr_global": 0.9032163289605386, + "snr_corrected": 0.9032163289605386, + "layer_snr_mean": 0.8835472645906415, + "mean_band_energies": [ + 3.953065952777519, + 3.4384359982509523, + 4.1422359481491595, + 4.268165305996948, + 4.632941612358489, + 5.441545099392218, + 5.980114131083787, + 7.1059112606126495, + 7.826006958083158, + 8.531433813431297, + 8.994599044934718, + 8.738993894586562, + 9.026195495278385, + 9.391076977074082, + 11.239765205132842, + 14.578905901453771, + 4.019639411852872, + 3.547412737584139, + 4.14686690299469, + 4.096405410445768, + 4.715394478028094, + 6.559043221502086, + 5.506484901626362, + 8.051860459712595, + 6.994483263806611, + 7.43585016604469, + 7.714063071480374, + 8.919990325941168, + 8.248087209461895, + 9.285694310590683, + 14.386049072365926, + 11.456127235124493 + ], + "layer_snrs": [ + 0.9793584804934982, + 0.8763143955445245, + 0.8415403359194379, + 0.8880745215498504, + 0.8936372075488783, + 0.8988820321930456, + 0.8455459279067038, + 0.9174679267556224, + 0.8857812187230721, + 0.8881125626920003, + 0.8804814500507204, + 0.8686871886734093, + 0.8695513110118298, + 0.8388153746485172, + 0.8515448048796498, + 0.862703839852447, + 0.8196578026448774, + 0.8623651498475126, + 0.9022014077477166, + 0.9215660790087216, + 0.8654169316595541, + 0.9131334971786473, + 0.8981551244665501, + 0.9361397791786076 + ] + }, + { + "model": "Qwen/Qwen2.5-3B", + "short": "Qwen2.5-3B", + "theta": 1000000, + "T_train": 8192, + "n_heads": 16, + "n_kv": 2, + "d_head": 128, + "n_layers": 36, + "k_dead": 36, + "n_pairs": 64, + "gamma_rand": 0.964, + "gamma_text": 0.772, + "snr_global": 0.8806074396662721, + "snr_corrected": 0.8806074396662721, + "layer_snr_mean": 0.8779535893465105, + "mean_band_energies": [ + 8.258374764617118, + 8.530581014978711, + 9.0921296208914, + 9.74542850871339, + 10.371327689217063, + 11.261751124148692, + 11.71900305074188, + 12.581573111282864, + 12.948883425581554, + 13.546673103479739, + 14.056310061908352, + 15.298073682638128, + 15.733690931548734, + 16.799295038545257, + 17.732245044596723, + 18.440038135491903, + 19.06599694553063, + 19.94782929088912, + 21.022966521857015, + 21.640077361154017, + 22.013505968683077, + 23.746046001497056, + 23.814673884287956, + 24.382287336727906, + 24.793978182098225, + 24.047625035665675, + 25.73469192577065, + 25.090471547866112, + 25.08550830014289, + 23.762961901493114, + 23.346760990650456, + 23.34272611501102, + 8.114272662849654, + 8.503591329382498, + 9.29552959483764, + 10.02337132649432, + 10.755936938695564, + 11.43039146612689, + 11.35037791111829, + 12.920628461773065, + 12.937383417807848, + 14.074980798794932, + 13.948108792425352, + 15.41086812106611, + 15.696963239944687, + 16.754686477587907, + 17.442182821265224, + 18.050304410411844, + 19.205301634052635, + 20.025494927580738, + 20.888260829595882, + 21.874271581998087, + 22.33561786918985, + 23.59895934718369, + 23.608095262430194, + 24.358027585563537, + 24.255542115442992, + 24.385263006623724, + 25.257572677137315, + 25.08432064967099, + 24.191150739594768, + 24.077838579577005, + 23.47843496437321, + 23.92530407936351 + ], + "layer_snrs": [ + 0.8956451721489964, + 0.9294661349376346, + 0.9374797848466799, + 0.8784490078780575, + 0.8986553151908518, + 0.9111804532089779, + 0.8444166244245765, + 0.9019454843726556, + 0.9013127858909121, + 0.9266468106212808, + 0.8988197360501505, + 0.8886783439886468, + 0.8459588558832479, + 0.8605977438740229, + 0.8742748593403651, + 0.872370043078339, + 0.8812653405373329, + 0.8655064910830728, + 0.8935768831052082, + 0.8400306349943036, + 0.8228604450323329, + 0.8409152894854571, + 0.841861232954458, + 0.8817057334009857, + 0.8694221842535114, + 0.8249895147391799, + 0.8843177330393549, + 0.8490698111475532, + 0.8856938116930246, + 0.8692375161200255, + 0.8678331376707393, + 0.8974064675935306, + 0.859287875029756, + 0.8356666953169407, + 0.9136962459148364, + 0.916089017627379 + ] + } + ] +} \ No newline at end of file diff --git a/data/exp_wqk_spectral/pythia-1.4b_wqk_spectral.json b/data/exp_wqk_spectral/pythia-1.4b_wqk_spectral.json new file mode 100644 index 0000000000000000000000000000000000000000..35081335707b0bcba21f307a7dd9f525dc3860e5 --- /dev/null +++ b/data/exp_wqk_spectral/pythia-1.4b_wqk_spectral.json @@ -0,0 +1,4551 @@ +{ + "model": "EleutherAI/pythia-1.4b", + "short_name": "pythia-1.4b", + "theta": 10000, + "T_train": 2048, + "d_head": 128, + "n_pairs": 64, + "k_dead": 46, + "global_mean_snr": 0.8848770275655095, + "global_min_snr": 0.8417170421027536, + "global_max_snr": 0.9278749628125329, + "layers": [ + { + "layer": 0, + "mean_snr": 0.8755652009170405, + "per_head_snr": [ + 0.870407460606457, + 0.8489390080670177, + 0.9478509070158853, + 0.8706158175364282, + 0.8692130904531779, + 0.873275599775479, + 0.863055802329445, + 0.9050805970897757, + 0.8657456697767891, + 0.8767481408422946, + 0.8670450106536335, + 0.8531422874707344, + 0.8399731406040442, + 0.8929294410069966, + 0.887835219556773, + 0.8771860218877161 + ], + "mean_band_energy": [ + 0.5749303401534842, + 0.4687856493772071, + 0.49825970920939167, + 0.47117061712770303, + 0.4721474862809343, + 0.7941972336158576, + 0.9133694928074847, + 1.0221900738019976, + 0.5581644848988934, + 0.4594110885801929, + 0.48942113552122035, + 0.49559580062174646, + 0.619005850999419, + 0.7657663461422977, + 0.8404098598558947, + 1.0078694731266449, + 1.0364309121430786, + 1.0317708652376407, + 1.0481461926102826, + 1.0450676326923167, + 1.0417606280895164, + 1.0244777124935907, + 1.0370287326281986, + 1.0341327574508241, + 1.0440044050925852, + 1.0504478750472113, + 1.0282640646222176, + 1.043483548818792, + 1.0222960477692946, + 1.0466729012011524, + 1.0497623536021263, + 1.0278826304913677, + 1.0328985727158049, + 1.0279446368414926, + 1.0313610663087334, + 1.0135953361198728, + 1.0220137187825646, + 1.0478369255878186, + 1.0565190500655477, + 1.0478992646739311, + 1.028064775123142, + 1.0508273704274729, + 1.022455690849222, + 1.0461136538696234, + 1.033817630352165, + 1.0265371082625252, + 1.0311239293313825, + 1.0325580647731805, + 1.0270128241479732, + 1.03680877753153, + 1.0291716240782085, + 1.0373058017621553, + 1.0435719788161275, + 1.050398280824895, + 1.0573971134138096, + 1.043218402562049, + 1.0328225738194534, + 1.0394881985548883, + 1.0284916679242573, + 1.0301626424362744, + 1.0336818814117714, + 1.0343548769464286, + 1.0485079258095504, + 1.0355182351383594 + ], + "dft_magnitude": [ + 60.221803501372776, + 5.551848402837991, + 4.1200657131648954, + 2.054332496551108, + 0.14382148789330462, + 1.4640954993095712, + 2.3957803381721288, + 2.422984938765649, + 2.0873333499309124, + 1.2824579006570738, + 0.8717812164957947, + 0.3936342116928162, + 0.2547162215460349, + 0.3462934474026913, + 0.7116591377818772, + 0.7807950855739303, + 0.7570774817209476, + 0.6010579860101777, + 0.48136178554262427, + 0.3853761291169619, + 0.2134755584903558, + 0.11938703044742217, + 0.34383372992403305, + 0.5740480864820966, + 0.432067389145425, + 0.3706543782653257, + 0.2567538689726746, + 0.07599359764543945, + 0.08354173809689536, + 0.2507375371067123, + 0.35501778778395693, + 0.4107159169282045, + 0.557176062905306 + ], + "mean_band_dot": [ + -5.119632049854772e-06, + -3.284832267524962e-06, + -1.815516647241111e-06, + -2.1988319929278077e-06, + -1.2985020347855425e-06, + -1.3201875034951627e-05, + -2.3741555335021758e-05, + -2.2455781504504557e-05, + -6.912253995849937e-06, + -9.536195477721776e-07, + -2.0584475990403917e-06, + -5.332004793245915e-06, + -9.05367726034001e-06, + -1.1687438004059914e-05, + -1.4821104636908444e-05, + -2.2266701328987892e-05, + -2.1597191476985245e-05, + -2.146566998817434e-05, + -2.3135060615686598e-05, + -2.3220066054818744e-05, + -2.3506888794599945e-05, + -1.7033174685820995e-05, + -2.4123626218397476e-05, + -2.1764629808629365e-05, + -2.0504359213191492e-05, + -2.2838724895279938e-05, + -1.9318251275990406e-05, + -2.2334821061065213e-05, + -2.21769769517266e-05, + -2.346362170158045e-05, + -2.3743128025444094e-05, + -1.983606821909234e-05, + -1.988916326922663e-05, + -2.194700691404705e-05, + -2.1134983171577915e-05, + -2.0770237192380137e-05, + -2.1189768261820063e-05, + -2.220673033548337e-05, + -2.3338632843206142e-05, + -2.1540097122851876e-05, + -2.025655946624738e-05, + -2.3702935664005054e-05, + -2.0008357921597053e-05, + -2.131803978500102e-05, + -2.2758516934118234e-05, + -2.313563135203367e-05, + -2.1071753181445274e-05, + -2.3819190708707083e-05, + -2.023111679250178e-05, + -2.3045031841917307e-05, + -2.167721513046672e-05, + -1.993191413873774e-05, + -2.0528539579345306e-05, + -2.040318466356439e-05, + -2.140504193448578e-05, + -2.099978379987988e-05, + -2.1818630841607956e-05, + -1.956133274916283e-05, + -1.9332148710304864e-05, + -2.128127437117655e-05, + -1.7573268678461318e-05, + -2.0773343834434854e-05, + -2.2689417299659453e-05, + -2.0996076017354426e-05 + ] + }, + { + "layer": 1, + "mean_snr": 0.9052649213103889, + "per_head_snr": [ + 0.8334751130350064, + 0.9094621078102783, + 0.8783942324350169, + 0.8944919510713489, + 0.9601178237031274, + 0.9107863638801987, + 0.8954767337729708, + 0.8976427231017463, + 0.9244517900884576, + 0.9073634989893383, + 0.9204411399416614, + 0.962706891099873, + 0.9391301788306913, + 0.877291204689166, + 0.9023045347562622, + 0.87070245376108 + ], + "mean_band_energy": [ + 1.3616520754012795, + 1.0159878742987625, + 0.9689683954202521, + 1.1190299596172548, + 1.3706375955906176, + 1.7302814873299337, + 1.6396861456084897, + 2.031405852896695, + 1.2987743981148852, + 0.9783439892154633, + 1.2854190372311134, + 1.1545580639432458, + 1.2930560024043376, + 1.8136915032756487, + 1.7607787820720824, + 2.049564436651342, + 2.0152803706052405, + 2.0183944520752224, + 2.0443603221555113, + 2.0370055157303195, + 2.0281832652358975, + 2.029588757297259, + 2.0479581147312818, + 2.016287643562501, + 2.0470331609869685, + 2.062634977023296, + 2.024777592368488, + 2.034974662193405, + 2.0385819267788534, + 2.0390573457225796, + 2.0175047486803863, + 2.0254421421319897, + 2.0196270084624923, + 2.038368085075836, + 2.019332402405791, + 1.9978621251807356, + 2.0392971878141593, + 2.0412298043354182, + 2.0333467257182214, + 2.0510713293018523, + 2.0369333026603567, + 2.038234984173807, + 2.02033706648444, + 2.0436329791703236, + 2.0236044739019707, + 2.0497068667038785, + 2.0403429324658378, + 2.0315620304908757, + 2.0410192264578377, + 2.043115488765609, + 2.0198693378932067, + 2.045452418004218, + 2.0248901179781513, + 2.0263920460883096, + 2.0544428042217433, + 2.0170287929322788, + 2.0493657390027016, + 2.041573289499398, + 2.0190073602239504, + 2.016049776675001, + 2.03826070611861, + 2.0344076619756537, + 2.0310750638765254, + 2.031478483172264 + ], + "dft_magnitude": [ + 120.45681821358205, + 8.842640833737006, + 6.5779551445608435, + 3.3587781301599353, + 0.36838200912739494, + 2.294350040914231, + 3.858317721291848, + 4.058707641184553, + 3.58595673492705, + 2.469029294069017, + 1.499775151368943, + 0.6795919417375921, + 0.31761496761378955, + 0.3084429734596504, + 0.4443297404697619, + 0.787883332217669, + 0.7632599666426458, + 0.9584224300300298, + 0.7551026139884797, + 0.5133252338438745, + 0.4242446211690635, + 0.7092833075589302, + 1.380127565167409, + 1.3377232173453113, + 1.1483978159661719, + 0.965915525121454, + 0.541491042316286, + 0.3459130528679998, + 0.19409224451118828, + 0.2326064044194575, + 0.5464522105613621, + 0.705444990046944, + 0.9500114354386966 + ], + "mean_band_dot": [ + -1.436894201845007e-05, + 8.439155749329075e-06, + 8.117603400137341e-06, + -3.402907410077205e-06, + -5.3464963940541566e-06, + -1.3638159970241759e-05, + -4.0796606469939434e-05, + -4.0655760205510205e-05, + -1.591847970416893e-05, + 6.209368081044886e-06, + 1.1465466855042905e-05, + -5.4150916710682395e-06, + -1.2598900696048076e-06, + -1.3234252236316024e-05, + -4.7751212605362525e-05, + -4.066474546959853e-05, + -4.486835163675096e-05, + -4.022330126929319e-05, + -4.392012201037687e-05, + -4.022527537017595e-05, + -3.632895098348854e-05, + -3.426999580824486e-05, + -3.796826285906718e-05, + -4.5769175585519406e-05, + -4.645326595920096e-05, + -4.115628927081616e-05, + -4.5667992367270926e-05, + -4.1627107705721755e-05, + -4.090250781985105e-05, + -3.882752503159281e-05, + -4.182476688185943e-05, + -3.6765407628536195e-05, + -3.470404999461607e-05, + -4.034721342094372e-05, + -3.724854008169132e-05, + -4.223708756967426e-05, + -3.80704061058168e-05, + -3.720856278732754e-05, + -3.7798690485146835e-05, + -3.8548054078546556e-05, + -3.7049140757972054e-05, + -3.886038609834941e-05, + -4.305501184376226e-05, + -4.552624636033897e-05, + -4.53242042368629e-05, + -4.581540127901462e-05, + -4.109661830398181e-05, + -3.615158276204511e-05, + -3.956131277504937e-05, + -3.899898564441173e-05, + -4.1443157442699885e-05, + -3.916410025794903e-05, + -3.815291826470002e-05, + -4.539764580613337e-05, + -4.0918447247406675e-05, + -4.13118011692859e-05, + -3.896010957760154e-05, + -4.5915089032178e-05, + -3.2509449553685954e-05, + -4.031933416470679e-05, + -4.2886051588197915e-05, + -3.656177828759155e-05, + -4.267763392817869e-05, + -4.306765484329844e-05 + ] + }, + { + "layer": 2, + "mean_snr": 0.9057507232532152, + "per_head_snr": [ + 0.8991126869227261, + 0.9374554653804354, + 0.8879058607763588, + 0.927081348479935, + 0.9425556637223631, + 0.8494183310140434, + 0.9245969715792813, + 0.8167436605946181, + 0.9208708044611695, + 0.9050351643436308, + 0.9047485775301899, + 0.8449972574597336, + 0.9008218999987561, + 0.9261011123046972, + 0.8920400700616132, + 1.0125266974218905 + ], + "mean_band_energy": [ + 1.4456925194369195, + 1.2611134165597921, + 1.3207220931032546, + 1.2464038362598533, + 1.119372529789774, + 1.5741752138084317, + 1.6786614383065412, + 2.100275140614091, + 1.3833705601602886, + 1.2630845335530947, + 1.2645119398751765, + 1.22880792520483, + 1.3211964702676209, + 1.5616931046981704, + 1.839369478441204, + 2.11787961489608, + 2.1183533685716465, + 2.189269530631114, + 2.1377569932988254, + 2.166353639875722, + 2.1605615416883692, + 2.1734495689429982, + 2.114254402080363, + 2.1108638588391253, + 2.13587576924866, + 2.1429322200600858, + 2.1368813881187787, + 2.0342987368495606, + 2.134638070806357, + 2.1589100345572634, + 2.130552891244938, + 2.121430787991778, + 2.0604482112522007, + 2.120767418991961, + 2.1529189782778926, + 2.167340923947834, + 2.134119756500664, + 2.1687751130469115, + 2.075556206023852, + 2.108322401268353, + 2.1578724727824268, + 2.11761789471655, + 2.15078169209041, + 2.1298521613935644, + 2.169278922341695, + 2.1393415497822117, + 2.13001349170087, + 2.0992453200634857, + 2.1520731386605156, + 2.164371139042413, + 2.1624760364852014, + 2.1307080521515864, + 2.0414313740365113, + 2.1530943625086296, + 2.1124830915146617, + 2.0757461078680057, + 2.124736317412245, + 2.159443808637479, + 2.0792982080625393, + 2.1512823548533104, + 2.1142683086414253, + 2.144129875830956, + 2.189587206247572, + 2.1025331010959043 + ], + "dft_magnitude": [ + 126.03262761501055, + 9.401965009778255, + 6.941650948839809, + 3.8975172028429412, + 0.4616387753991988, + 2.4405261597337815, + 3.7061686129388116, + 3.705677196959642, + 2.657688120627444, + 1.8998888579581708, + 0.5416923490234523, + 0.17810326926829792, + 0.4086504679382293, + 0.4100364581764028, + 1.3403204344833985, + 1.4565325978412593, + 1.08332012439505, + 1.1102879896887101, + 0.654811189321079, + 0.4554204205137503, + 0.3810566299967696, + 0.4882700438249076, + 0.7243004546421926, + 0.5656216471721294, + 0.7658260359937658, + 0.6722100513962139, + 0.59094895024725, + 0.5120147289996163, + 0.6396267058873393, + 0.29867806929854557, + 0.5966876650340481, + 0.9334944970218957, + 1.1343978820717524 + ], + "mean_band_dot": [ + -3.137422322652128e-05, + -9.675232689687618e-06, + 6.316039744547197e-06, + 6.096952347434126e-07, + 5.721472433606323e-06, + 1.989293927806557e-05, + 2.8779961496638862e-05, + 8.993909638377318e-05, + -2.006067913384868e-05, + -2.26682590209748e-06, + 4.448850702942764e-06, + 5.782521327546419e-06, + 8.478328766159393e-06, + 1.835476177802775e-05, + 3.971528388646561e-05, + 8.652883607851436e-05, + 7.911518071068713e-05, + 8.419711735996316e-05, + 8.742770179992476e-05, + 8.088667767935931e-05, + 9.056537373908213e-05, + 8.449293221701737e-05, + 8.023704633330908e-05, + 8.728393501655773e-05, + 7.977213615362189e-05, + 9.372396388585003e-05, + 8.177770686756958e-05, + 7.437415561639682e-05, + 8.450581106700383e-05, + 9.321164546349792e-05, + 7.581531279043929e-05, + 8.791946045505483e-05, + 7.858259913717802e-05, + 7.727913437349798e-05, + 9.122637982272863e-05, + 8.591607917196598e-05, + 8.839949899197563e-05, + 8.891437050806417e-05, + 8.615403092449014e-05, + 8.05617077048737e-05, + 9.327941194214873e-05, + 8.710496854291705e-05, + 8.497486629721607e-05, + 8.62108895489655e-05, + 9.264044342671696e-05, + 9.427314039811563e-05, + 8.156269227299617e-05, + 8.022793709017151e-05, + 8.583060457567626e-05, + 8.885195478569585e-05, + 8.749398033103262e-05, + 9.16494324485484e-05, + 5.788259679206931e-05, + 9.151322944944695e-05, + 8.644944202274019e-05, + 7.682665784614073e-05, + 8.63098984424937e-05, + 8.963953561647031e-05, + 7.865569804721417e-05, + 8.913120370834804e-05, + 8.158564955351721e-05, + 8.980975748329456e-05, + 8.602010601066468e-05, + 8.681362236018231e-05 + ] + }, + { + "layer": 3, + "mean_snr": 0.9059193048271534, + "per_head_snr": [ + 0.9437515540066922, + 0.876905744560291, + 0.9176133384962696, + 0.9435003700422341, + 0.9523810953173351, + 0.933009259888507, + 0.8933132027065854, + 0.8647955306865418, + 0.9363075617657798, + 0.9208549107892837, + 0.906425951657305, + 0.897099021477993, + 0.8900414767384298, + 0.8060654655023665, + 0.8921106540070846, + 0.9205337395917573 + ], + "mean_band_energy": [ + 1.394204607394916, + 1.2801150279963074, + 1.3522271559321521, + 1.3073351378278026, + 1.4768660457341074, + 1.9021974338379053, + 1.8999494482790773, + 2.289297283417614, + 1.453941809298203, + 1.2883725770791379, + 1.4125936965317765, + 1.54879856065474, + 1.6838561942960641, + 1.8527690853891339, + 2.0426728654186825, + 2.2563834958820292, + 2.2246521048477677, + 2.2284277779003485, + 2.197245622652506, + 2.3156529054466413, + 2.3035826568673228, + 2.3181003505989928, + 2.330290405757464, + 2.301014308642505, + 2.306196854359504, + 2.2731312140139446, + 2.287005413504697, + 2.2803000851049875, + 2.2877106252320285, + 2.243429589568211, + 2.2776200328627603, + 2.2915220042729896, + 2.334946650288419, + 2.2957376890982912, + 2.2071663616440036, + 2.2726246503028342, + 2.2330849680366285, + 2.222090153463979, + 2.321515700345465, + 2.2868739667701057, + 2.3037961007565366, + 2.305628467182186, + 2.260390736525788, + 2.2413035845815825, + 2.307862574026341, + 2.2786660887696932, + 2.2758037022452227, + 2.296769032767628, + 2.273094497243904, + 2.331775163334596, + 2.2974558170678456, + 2.321917709993814, + 2.2877916924472084, + 2.2997445400068557, + 2.258334773971283, + 2.306880805746836, + 2.2753341179872013, + 2.266533162368316, + 2.2986813151359895, + 2.270346482744876, + 2.2544302558877263, + 2.298500745649455, + 2.313617407915922, + 2.262732061956152 + ], + "dft_magnitude": [ + 135.968893352865, + 9.212564199510558, + 6.984905153041756, + 3.205536362358911, + 0.5716282214332525, + 2.302103198100936, + 3.5456376005232597, + 3.805775611545437, + 3.445887095786837, + 2.3072624586858557, + 1.412658110359926, + 0.5681528654682373, + 0.1570445644552762, + 0.6626079525241735, + 1.1076426146378153, + 1.6040680021551226, + 1.3246034046632258, + 0.8450171242225688, + 0.4529768823209625, + 0.2614540083698979, + 0.20211907059835652, + 0.8102321845460159, + 0.6776609542906947, + 1.2010194835230685, + 0.9293267003743205, + 1.013737566891157, + 0.6275090524993288, + 0.3977227577566354, + 0.42988993484769994, + 0.3826841172968923, + 0.5249368037401039, + 0.9960324618926526, + 1.1010489318759795 + ], + "mean_band_dot": [ + -2.1739457196190415e-05, + 6.019324132466863e-07, + 8.657195024852626e-06, + 2.0080901705910037e-06, + 1.5801590507180663e-06, + 8.225141243656253e-06, + -1.276401550853734e-05, + 1.4786239603381546e-05, + -2.5163620833268396e-05, + 2.974388038978759e-06, + -3.423518933232117e-06, + -1.3635566347147687e-06, + -3.5009286776244153e-06, + 1.0451692062929396e-05, + -9.305889761890285e-06, + 1.0614324800428676e-05, + 1.4857833022574596e-05, + 1.1141031386330269e-05, + 1.6420391787619337e-05, + 1.7744852584655746e-05, + 1.5683377000641485e-05, + 2.221203854446685e-05, + 1.9041302380173875e-05, + 1.4668089008296192e-05, + 1.5706755675637396e-05, + 1.6709258659375337e-05, + 1.8629620910814992e-05, + 1.434156374102713e-05, + 1.3963509047698608e-05, + 1.2155751889508792e-05, + 1.7321813118087448e-05, + 1.9648363718260953e-05, + 2.125826401311315e-05, + 1.8314930869678392e-05, + 1.1464635761626596e-05, + 1.6774478538650328e-05, + 1.2453764554720692e-05, + 1.623173696430058e-05, + 1.7736133315793268e-05, + 1.057057178144305e-05, + 1.4851019003003785e-05, + 1.7657692183092877e-05, + 2.2050699584497124e-05, + 1.3965213312872038e-05, + 1.6519647829227324e-05, + 1.626780124297511e-05, + 1.801533946377276e-05, + 1.613548027989964e-05, + 1.5704687086781632e-05, + 1.6087978679024673e-05, + 2.154618840677358e-05, + 1.5227458419531104e-05, + 1.9470770155294304e-05, + 2.1359356964012477e-05, + 1.699664734644557e-05, + 1.954838072038001e-05, + 2.0979370333407132e-05, + 1.6624882078986047e-05, + 1.8989289898740935e-05, + 1.4221111342749282e-05, + 1.650819710619089e-05, + 1.581954592211332e-05, + 1.532324290565157e-05, + 1.9176869130888008e-05 + ] + }, + { + "layer": 4, + "mean_snr": 0.892219245129956, + "per_head_snr": [ + 0.8593099380529318, + 0.8218490379409334, + 0.8840765611984174, + 0.8811244968775177, + 0.9043707850952709, + 0.9461501748557706, + 0.975161140299676, + 0.8122300008747372, + 0.8595698715197324, + 0.8818736892741834, + 0.8988413552283729, + 0.8631927086568824, + 0.9269427868483741, + 0.869221724771392, + 0.9213016666615416, + 0.970291983923561 + ], + "mean_band_energy": [ + 1.1349187124910511, + 1.144822477295134, + 1.26894779904282, + 1.4097709083898486, + 1.528508335182066, + 1.711046804005738, + 2.0393322387546577, + 2.4686130053322977, + 1.137329656143686, + 1.176739302626452, + 1.2883566158998663, + 1.2797958252188186, + 1.5001228697351947, + 1.779909163648163, + 1.9865374337271022, + 2.4341830963784825, + 2.4746887596034903, + 2.439524633951444, + 2.4189376606858426, + 2.3836648628925166, + 2.4163860174852694, + 2.3143023386972477, + 2.4308511080396107, + 2.387975404938592, + 2.402046693040777, + 2.445606456845356, + 2.412660353372306, + 2.384759991094705, + 2.430608105609666, + 2.422365637654657, + 2.3797170122008984, + 2.411281502012989, + 2.3755153043984234, + 2.4440481177854965, + 2.4472635944869694, + 2.3465064161353313, + 2.3784827124574095, + 2.421092542009773, + 2.372754986707288, + 2.3782545440977305, + 2.4710559648158386, + 2.4584904311036393, + 2.3956941191243333, + 2.44245660910813, + 2.32955992181635, + 2.4806431195530445, + 2.400603991822256, + 2.3312907528951294, + 2.397338514994598, + 2.4011502796363247, + 2.33298305876015, + 2.4664723145928886, + 2.3884171124602664, + 2.4496336793521554, + 2.39144860137775, + 2.410254206006499, + 2.299011222460404, + 2.3590661099574137, + 2.433308868968484, + 2.4583463361141416, + 2.398764900878493, + 2.476483793907793, + 2.4468600187285343, + 2.455692593563249 + ], + "dft_magnitude": [ + 140.88325552207303, + 11.835993823644618, + 8.62020792622013, + 4.544623824544583, + 0.21163528901556905, + 3.228504065083631, + 5.23731687416297, + 5.627594102618294, + 3.777716388128321, + 2.1523283606908294, + 0.9678374591399505, + 0.8272979588057996, + 0.15871077419174184, + 0.911880529882224, + 1.799771326358861, + 2.6554946246346898, + 2.058605131920301, + 1.9176175212076978, + 1.0926814192272212, + 0.62013977703449, + 0.4540579482191249, + 0.786242478999014, + 1.434282123658097, + 1.6677491998708553, + 1.5929225152122497, + 1.6728106853454647, + 0.9130796812791322, + 0.13874074343566972, + 0.2418844406950572, + 0.4660434224969807, + 0.8689306303023893, + 1.3770024002184336, + 1.8652309915293301 + ], + "mean_band_dot": [ + -3.667229746895373e-05, + -1.4987124998810941e-05, + -1.4386276490085947e-05, + -2.1637154517861745e-05, + -1.636835951046578e-05, + -1.3414032622449668e-05, + -9.568180615815436e-06, + 3.824566755383785e-05, + -3.3242084384710324e-05, + -1.479820343774918e-05, + -4.342243187238637e-06, + -1.720151330530939e-05, + -2.108514708254461e-05, + -4.258846786342474e-07, + -2.291812080557065e-06, + 2.48356855934162e-05, + 3.773587972943915e-05, + 4.1969116523432604e-05, + 4.7524360752504435e-05, + 4.002927758506303e-05, + 3.10883687433261e-05, + 2.9914508559159003e-05, + 3.380809994979472e-05, + 3.324902556300913e-05, + 2.8963571480744577e-05, + 3.093290888500633e-05, + 3.784308682952542e-05, + 2.9429324740704033e-05, + 2.9043218717106356e-05, + 3.1142373359216435e-05, + 3.5140996658356016e-05, + 4.039584467818713e-05, + 3.491934824495502e-05, + 3.195609991735182e-05, + 3.613702330085289e-05, + 3.328088263288009e-05, + 5.0102274826713256e-05, + 3.440692583467353e-05, + 2.924246648206008e-05, + 3.6838534100525067e-05, + 3.842728743563839e-05, + 2.9501446618951377e-05, + 3.0338449903410947e-05, + 2.9664557473552122e-05, + 3.218973688490223e-05, + 3.725057263181952e-05, + 3.977599061499859e-05, + 2.514680926424262e-05, + 3.9064518574605245e-05, + 3.2185684169405704e-05, + 3.665922974960267e-05, + 3.161063118284346e-05, + 3.385896621921347e-05, + 2.9894858698753524e-05, + 2.8187615662034204e-05, + 3.5356472892544843e-05, + 2.1176424297664198e-05, + 2.3666688207413245e-05, + 4.211013200006164e-05, + 3.839174382846977e-05, + 3.4466693044521435e-05, + 3.6624217159442196e-05, + 3.6550978691707314e-05, + 3.136778491352743e-05 + ] + }, + { + "layer": 5, + "mean_snr": 0.9064631968781224, + "per_head_snr": [ + 0.9399556714709476, + 0.8993388718394245, + 0.9165397683357001, + 0.9207047056742698, + 0.7918786032927871, + 0.898590789971879, + 0.9465066143760646, + 0.8963008533352547, + 0.8135410802823715, + 0.935300660881545, + 0.8954105839561406, + 0.843288940328706, + 0.9512464990734509, + 0.8557968257825519, + 0.8983526774243775, + 1.1006580040244882 + ], + "mean_band_energy": [ + 1.3057042583051566, + 1.2633000173913245, + 1.3694252998374596, + 1.4481870695782124, + 1.5321825376098024, + 1.8794382619775627, + 2.2376418929238806, + 2.5594802098535827, + 1.3094966716334788, + 1.3128582729546867, + 1.4192465459753603, + 1.407706785139851, + 1.5065271579229251, + 1.6747302912158872, + 2.0867191992999077, + 2.5540514083899972, + 2.5093921094471665, + 2.4455121833219025, + 2.4788053310137674, + 2.5618113120545125, + 2.5172506019732546, + 2.549586600412165, + 2.5724362255412005, + 2.462204504872446, + 2.4762766653959014, + 2.5621882029957166, + 2.6097199408859053, + 2.475043448694818, + 2.4951851294668104, + 2.5327332264127422, + 2.5300446517360378, + 2.451596112516336, + 2.4673912148365025, + 2.571223911642109, + 2.622638032777978, + 2.4961866251093667, + 2.551982567020469, + 2.60745297695523, + 2.5941790541704295, + 2.5683898959137337, + 2.5907857090757647, + 2.583562817678697, + 2.6447790321704323, + 2.6084205861008014, + 2.6080970700628767, + 2.565867404386056, + 2.495211896358252, + 2.6155158365372584, + 2.506030551348922, + 2.509361088333542, + 2.5091930310305894, + 2.5906686065455893, + 2.4311583339395995, + 2.4336676528047843, + 2.401903139788706, + 2.563023183465517, + 2.529854349083938, + 2.6145157365103593, + 2.5249843620010513, + 2.498477656966214, + 2.5830908684284304, + 2.6003014215712206, + 2.579664605872972, + 2.5868848714007275 + ], + "dft_magnitude": [ + 148.75094621663789, + 12.779072142314478, + 8.13302686935727, + 4.89912643736617, + 0.5875470438009824, + 3.851007956297999, + 5.473471942565323, + 5.250244564327144, + 3.641582688737188, + 2.7504756645166846, + 1.3478758614783115, + 1.0892423099349766, + 0.3048503609918311, + 1.0284984230535672, + 2.315526391901779, + 1.9842241999114103, + 2.4731526139399014, + 2.195444204877774, + 0.4988675977050922, + 0.7593173207450646, + 0.5237613319314615, + 0.3272639421912562, + 1.1682260381682767, + 1.2774435846391312, + 1.7478935431817832, + 1.292865851051686, + 0.7982938482893706, + 0.2932557337813386, + 0.3270800188198975, + 0.4975072440694278, + 0.47410434514217464, + 1.4424304735586595, + 1.5569501427680166 + ], + "mean_band_dot": [ + -4.3504540634842215e-05, + -1.3641283601373289e-05, + -3.336040833801235e-06, + -1.315652724542815e-07, + -5.822778604169798e-06, + 1.38362340464937e-05, + 2.843172291022711e-05, + 7.107019663976644e-05, + -3.8279560250487066e-05, + -1.402060524924309e-05, + 4.993008367648599e-06, + -5.8382857233141294e-06, + -1.9244830014031322e-06, + -1.0366070554823636e-05, + 1.5161146507125522e-05, + 7.22469677754134e-05, + 7.19783210456626e-05, + 6.54292240938048e-05, + 7.431651928868632e-05, + 7.182604161926065e-05, + 6.561015007378046e-05, + 6.88866130644783e-05, + 8.263730990165641e-05, + 6.877920665715465e-05, + 7.105980265009748e-05, + 7.476084127233662e-05, + 6.936846055793922e-05, + 6.542901041939331e-05, + 6.662859277639654e-05, + 7.235612025624505e-05, + 6.314208158642032e-05, + 6.420054177169732e-05, + 6.034947154631709e-05, + 7.084680987645697e-05, + 7.654125873557405e-05, + 6.832382477739429e-05, + 8.085678148006537e-05, + 7.464441361548779e-05, + 7.505428412457604e-05, + 6.382279460837026e-05, + 7.481487702420964e-05, + 8.048880854971685e-05, + 7.495713947491822e-05, + 7.744301814227583e-05, + 8.19983674773539e-05, + 7.118810492556804e-05, + 6.87705474433642e-05, + 7.75071437715269e-05, + 6.640877223418329e-05, + 7.434767590552838e-05, + 7.878571244646082e-05, + 6.946096939941526e-05, + 6.487393957854692e-05, + 6.876346355966234e-05, + 4.9903703256859444e-05, + 7.878070533706705e-05, + 7.821389507967069e-05, + 7.598633891170437e-05, + 7.64271126456606e-05, + 6.904310502875433e-05, + 8.122846105607096e-05, + 7.763095430846079e-05, + 7.827173620000849e-05, + 7.916520695516738e-05 + ] + }, + { + "layer": 6, + "mean_snr": 0.9190336856083441, + "per_head_snr": [ + 0.8822839215783338, + 0.8827111551389327, + 0.9740451730876225, + 0.8995699320188524, + 0.8859549252479001, + 0.9265807419513216, + 0.9088887534939116, + 0.9215858894104895, + 0.9625093484081675, + 0.8720639566742024, + 0.9604657345415455, + 0.9371779076491235, + 0.906266935022611, + 0.9516022383618437, + 0.9146520651155252, + 0.9181802920331223 + ], + "mean_band_energy": [ + 1.4342373193379525, + 1.353391659427798, + 1.4101212344984058, + 1.5555640764242131, + 1.800483044350786, + 1.7455493513174565, + 1.926294822433861, + 2.2289052540743093, + 1.4296586352789955, + 1.3805910858460375, + 1.3760200356397498, + 1.4524490864358572, + 1.7062820667489405, + 1.880322514266283, + 1.8423828867777186, + 2.0777918992340894, + 2.196024723277346, + 2.184489893605468, + 2.248822090667229, + 2.2328292226206194, + 2.163085912304675, + 2.236819500887435, + 2.235551637978178, + 2.220188967328422, + 2.2532457990469537, + 2.2437415117038055, + 2.240936716148272, + 2.1990945842282263, + 2.241401751919767, + 2.23731497806903, + 2.1863014794624416, + 2.2309142497666086, + 2.245294355760154, + 2.1781799641178736, + 2.2440480784555703, + 2.212666011550999, + 2.2211304454191794, + 2.2285636849279147, + 2.190555341333077, + 2.265045917965601, + 2.2571032425601345, + 2.2242011689355348, + 2.2481768194296325, + 2.234005033670069, + 2.2331486282197757, + 2.2432790501232605, + 2.2223281306924303, + 2.2314965475556283, + 2.2111472119780418, + 2.2036348609503547, + 2.225589736778237, + 2.2354171816326307, + 2.2418748968416553, + 2.2349087819557587, + 2.213750510692046, + 2.2222120464400756, + 2.2162527173502573, + 2.22714451720437, + 2.2095922323325583, + 2.19849112059657, + 2.21001585844765, + 2.2438209071912425, + 2.2287467125481886, + 2.1470474797680117 + ], + "dft_magnitude": [ + 133.29967718453142, + 8.304012429085098, + 5.745863168323767, + 2.63627526599683, + 0.3846514793619913, + 2.2240748221089026, + 3.191254957597896, + 3.313867686733308, + 2.541709152148736, + 1.765024311640908, + 1.1542364222499422, + 0.5179819620842497, + 0.5122566731034359, + 1.0299490105763383, + 1.2549745116062117, + 1.0845609464427797, + 0.9178007868333412, + 0.7117892543873097, + 0.5277769737690674, + 0.2543556011246459, + 0.3968956688788593, + 0.7787784731691494, + 1.0396407234781766, + 1.143996856245745, + 1.114747862437095, + 0.6943564166719624, + 0.6386138291702504, + 0.36442113619837163, + 0.193631446504167, + 0.10622996028203571, + 0.38236761603827607, + 0.6528380308546068, + 0.6804670351116897 + ], + "mean_band_dot": [ + -2.4256145906065285e-05, + 9.064060257912843e-06, + 9.429238673419604e-06, + 6.269530814506652e-07, + -1.1379082707208e-05, + 1.4522539260042322e-07, + -4.0520153902434686e-05, + -7.225379874853388e-06, + -3.336652483199032e-05, + -1.234762507351661e-06, + 1.1908883976730067e-05, + 8.086282861086147e-06, + -5.555857356398519e-06, + -2.1314843181130527e-05, + -3.770670349467764e-05, + -1.0552321896284411e-05, + -1.1035586112484452e-05, + -1.6124302405273738e-05, + -2.111117098024806e-06, + -5.055643327978032e-06, + -1.0372226874721946e-05, + -2.974417029122378e-06, + -4.745706121411786e-06, + -5.839306567168023e-06, + -1.486012592977204e-06, + -9.948430488293525e-06, + -5.370725659759046e-07, + -8.431506898887164e-06, + -1.6866896274336796e-05, + -1.1610365760361674e-05, + -9.13456428719428e-06, + -5.926575465053929e-06, + -3.481632539603652e-06, + -5.269217190573272e-06, + -1.296665490713167e-05, + -7.06598305555417e-06, + -7.974827667567297e-06, + -5.699021492944212e-06, + -7.89645390852911e-06, + -4.92501743565299e-06, + -9.813990189400101e-06, + -9.01930570762488e-06, + -4.6023149025131715e-07, + -7.20555406985568e-06, + -7.795092066942289e-06, + -1.2099750620109262e-05, + -6.9742053767640755e-06, + -8.173427687552248e-06, + -7.285065336759544e-06, + -1.1029875992107918e-05, + -4.558961391509797e-06, + -5.078435322047881e-06, + -3.3026065864305565e-06, + -7.267975661306991e-06, + -1.3056290583790542e-05, + -1.0118987688656489e-05, + -3.1105128073249944e-06, + -5.48546591971899e-06, + -8.484738543756976e-06, + -1.4736170669493731e-05, + -6.4503346948185936e-06, + -8.512897970547328e-06, + -1.0520742222297486e-05, + -1.2154043616874333e-05 + ] + }, + { + "layer": 7, + "mean_snr": 0.8933636445732667, + "per_head_snr": [ + 0.9222063285702453, + 0.9185868576428811, + 0.8910852936995296, + 0.8090647737616027, + 0.8894998573779463, + 0.9626537023744212, + 0.8274690676801101, + 0.8873378345454012, + 0.9555008970783905, + 0.9154557984015611, + 0.8563484495957002, + 0.9125807709536348, + 0.878224228095533, + 0.8634306233889647, + 0.9291758346892233, + 0.8751979953171218 + ], + "mean_band_energy": [ + 1.1997549388162017, + 1.2429907551672579, + 1.420487683218502, + 1.437968825485701, + 1.661696930623996, + 2.042447024393784, + 2.173545886891085, + 2.483528732132574, + 1.109334077957568, + 1.2567195143047714, + 1.2901496877689898, + 1.4489743291492294, + 1.7595728600279466, + 1.9110862009420249, + 2.151078242068767, + 2.52203872536288, + 2.4641829938112556, + 2.5002354522866783, + 2.4798784312976956, + 2.5427704170113303, + 2.478812901309496, + 2.4589167120921447, + 2.515606457266183, + 2.5410545495273054, + 2.5381305042597715, + 2.476422891437376, + 2.5740638408288863, + 2.508231430672083, + 2.5761833486615187, + 2.5089858561557685, + 2.4525817003870403, + 2.46715829616618, + 2.5575228620917647, + 2.522724143977375, + 2.5425887487541923, + 2.4813571976605937, + 2.5592327776048505, + 2.538684824112387, + 2.5129931245428576, + 2.4953606121952134, + 2.508688687889502, + 2.566584108422652, + 2.4895490849754323, + 2.4436086044476024, + 2.540138208693076, + 2.5268695493713427, + 2.574037769366324, + 2.4885590005593556, + 2.5041253088447073, + 2.458801358354032, + 2.5322820135884205, + 2.4335085423968907, + 2.5465742739699913, + 2.4994859741378654, + 2.4485183986248256, + 2.549057427473432, + 2.5059633129245222, + 2.4857681276782593, + 2.521105000807534, + 2.552362059001328, + 2.52567464993502, + 2.524882296757796, + 2.5510771281455744, + 2.5550578320780293 + ], + "dft_magnitude": [ + 147.73733320686475, + 11.930275667919982, + 8.558459176248682, + 4.454034322697494, + 0.14897920933530118, + 3.145974954325911, + 5.0639423461589255, + 5.210891269858902, + 4.246864590107302, + 2.97818412941718, + 1.6459415244766704, + 1.1481405849668498, + 0.14529921600360088, + 1.0743327595320467, + 1.2332420615077928, + 2.5021846852494107, + 1.862134537251452, + 1.7045915967191199, + 1.1563814017529415, + 0.888206361565058, + 0.24293879887113495, + 0.7130644055743793, + 0.8606031229880997, + 1.529945877104982, + 2.1158895261795787, + 1.5255077683016207, + 1.245986016988593, + 0.4237536732705091, + 0.35566213600090035, + 0.2379690722785268, + 1.0773625619385234, + 1.7921593076929865, + 1.2070695349577534 + ], + "mean_band_dot": [ + -4.74808655326342e-05, + -2.6370408775733267e-05, + -4.863578084268738e-06, + 4.4103066443312855e-06, + -1.2046725004211112e-06, + -1.644134908929118e-05, + -1.0277909495925996e-05, + 3.859580436937904e-05, + -4.733752376750999e-05, + -2.5167382887048007e-05, + -3.179675573505847e-06, + 5.087429528316534e-06, + -1.5962587980311582e-06, + -8.848096342717326e-06, + 1.913205170467336e-05, + 3.721844450410572e-05, + 3.583274155971594e-05, + 3.905173591078892e-05, + 2.802646810096121e-05, + 5.068438770194916e-05, + 3.4976951440057746e-05, + 3.768472197407391e-05, + 5.61632459152861e-05, + 3.132400084382425e-05, + 5.269214956626911e-05, + 4.7928117282936e-05, + 4.848921963684916e-05, + 4.3228943468420766e-05, + 4.311703213488727e-05, + 3.76244063033937e-05, + 4.736457549370243e-05, + 5.04102632703507e-05, + 4.815290918713799e-05, + 4.1449409138749616e-05, + 4.280009488866199e-05, + 4.437342769847419e-05, + 4.073471214383062e-05, + 4.754476776724914e-05, + 4.074350356120249e-05, + 4.338496387390478e-05, + 4.634109535572861e-05, + 4.615748827063726e-05, + 4.7734422878420446e-05, + 2.2482436961013263e-05, + 4.7430270569748245e-05, + 5.369852780745532e-05, + 4.3720474877773086e-05, + 3.5211548492952716e-05, + 3.74119099433301e-05, + 4.0491979973467096e-05, + 5.563778279338294e-05, + 4.2535127022347297e-05, + 5.44714873740304e-05, + 3.579819582455457e-05, + 3.180010725145621e-05, + 3.807690194435054e-05, + 4.682098622765807e-05, + 4.5359651664966805e-05, + 3.721932003486472e-05, + 5.1454991535138106e-05, + 4.43090461317297e-05, + 3.7460165685843094e-05, + 5.48112409433088e-05, + 3.768609239784837e-05 + ] + }, + { + "layer": 8, + "mean_snr": 0.9278749628125329, + "per_head_snr": [ + 0.9398939742096408, + 0.9194917105714044, + 0.9291012811152559, + 0.9490119173682267, + 0.9905562393528685, + 0.9197734531021902, + 0.8678501014862227, + 0.8768961249512357, + 0.9020013832251846, + 0.9498143224185919, + 0.8875698685415372, + 0.9316343379982972, + 0.957597668751481, + 0.894466358546213, + 0.9611296025907793, + 0.9692110607713974 + ], + "mean_band_energy": [ + 1.2184137238160355, + 1.3216073277574476, + 1.4301487831715716, + 1.543844164630256, + 1.7299672051619472, + 1.8428820560292167, + 1.8888061216565886, + 2.113137067410378, + 1.195887112264039, + 1.321402121874186, + 1.4512245679045614, + 1.5351513641679122, + 1.7883497578635503, + 1.8216158916439666, + 1.9771064710073247, + 2.101859667858326, + 2.065689768297937, + 2.1279494128846563, + 2.1162370610729893, + 2.127348729110224, + 2.142043775122195, + 2.084234947576914, + 2.0876500322414753, + 2.1310166295161412, + 2.119856900690854, + 2.1384655526619234, + 2.125276911227818, + 2.123844232870593, + 2.147935639024901, + 2.1417636242025244, + 2.1137158675881333, + 2.1062906429356154, + 2.1477032577896207, + 2.136873025785542, + 2.1402043469313745, + 2.119344981042806, + 2.144119855097371, + 2.097332822647963, + 2.0601374199809133, + 2.129404891080129, + 2.1335475332150198, + 2.132698919763415, + 2.1377336004541743, + 2.1542966102246544, + 2.0925392336881914, + 2.14664689280543, + 2.14707068397989, + 2.091396677998543, + 2.126210177108692, + 2.1208039531557588, + 2.098806493508743, + 2.139727918287832, + 2.127570520498314, + 2.1090529323214735, + 2.1466127813935216, + 2.13665285213958, + 2.1045391139603775, + 2.146815859914935, + 2.1279100325323768, + 2.122527758934856, + 2.148148494140592, + 2.087536257512646, + 2.1388765204652698, + 2.114745864870195 + ], + "dft_magnitude": [ + 128.18831141447242, + 7.076541031946593, + 5.139583837443678, + 2.5357200091400194, + 0.25023011313816057, + 1.9757582199881965, + 2.810107626896067, + 3.201549871016769, + 2.5490728953090076, + 2.205130556144529, + 1.2527147810120896, + 0.6955271113369577, + 0.2040834065211589, + 0.5467430469001456, + 0.9786818065467083, + 1.5017499276687731, + 1.2633297014782885, + 1.0891300380644449, + 0.661435577818572, + 0.42040565357608967, + 0.22952162147839514, + 0.28318332723291856, + 0.9449670625177817, + 1.3653020527639035, + 1.4030750307426945, + 1.150897860093354, + 0.723468754386987, + 0.18096501519759076, + 0.26033619657577894, + 0.5089500104138189, + 0.6549853077302216, + 0.8330944265353328, + 0.948231888759679 + ], + "mean_band_dot": [ + -3.6843141046460914e-05, + -1.416397009990078e-05, + 2.7683617531693017e-06, + 4.725860009102689e-06, + -1.4592216842856942e-05, + -1.0826817629094876e-06, + -2.2340833567113805e-05, + 6.460038989075656e-06, + -3.801187088470215e-05, + -2.0420598403347867e-05, + -4.904409621531158e-06, + 1.7389771755915717e-07, + -1.4931960919284393e-05, + -5.167999233890441e-06, + -2.7331726897728004e-05, + 1.6069702354570836e-05, + 1.8030577280114812e-05, + 1.3274739984581174e-05, + 1.7327472278338973e-05, + 1.7524350141684408e-05, + 1.0837465339363916e-05, + 1.69081579883823e-05, + 1.875917826055229e-05, + 1.4259631257118599e-05, + 1.0396812058388605e-05, + 1.119106494229527e-05, + 2.0157317806024366e-05, + 1.7790389904348558e-05, + 1.2734773690681322e-05, + 1.7679454401786643e-05, + 1.631585001860003e-05, + 1.7356643439825348e-05, + 1.6023640569073905e-05, + 1.663438733601197e-05, + 1.3771215890301391e-05, + 2.123922229202435e-05, + 1.1541290938055226e-05, + 2.026362112417246e-05, + 1.1522069030434068e-05, + 1.521838214557647e-05, + 1.3333852450614359e-05, + 1.9022724984552042e-05, + 1.5594441876487508e-05, + 1.2824775524222787e-05, + 1.382344473199737e-05, + 1.733509594714633e-05, + 1.3778505376649264e-05, + 1.017231767264093e-05, + 1.3919942375650862e-05, + 1.5188676570687676e-05, + 1.9599803977143893e-05, + 1.831309202771081e-05, + 1.736132327323503e-05, + 1.7921319347635745e-05, + 1.87607757737851e-05, + 1.6651393750066745e-05, + 7.8315154610209e-06, + 1.5793507785133443e-05, + 1.2842022890424687e-05, + 1.3704068351216847e-05, + 1.2954031262779608e-05, + 1.0695148091599549e-05, + 2.101282561284279e-05, + 1.8102764265393034e-05 + ] + }, + { + "layer": 9, + "mean_snr": 0.8990020752974368, + "per_head_snr": [ + 0.9203232314280699, + 0.8500507540149158, + 0.9198897665533416, + 0.9017465029876944, + 0.9329335271849282, + 0.887246852231154, + 0.8937130763937935, + 0.9766327468351218, + 0.9250663830923044, + 0.8506366299082383, + 0.8338119812140716, + 0.9119128749427431, + 0.8583217988282582, + 0.9370109704479402, + 0.9122756090267508, + 0.872460499669664 + ], + "mean_band_energy": [ + 1.0683050265959313, + 1.1790603940190794, + 1.3267557230843456, + 1.5228287572583183, + 1.7576033036930867, + 1.9850260884782767, + 2.1378120125920437, + 2.4107391049364235, + 1.04155563325067, + 1.1678282257659278, + 1.394698012616171, + 1.4901399145852026, + 1.762842727905665, + 1.969514483488449, + 2.0636736573479264, + 2.4276799118702357, + 2.424175949433658, + 2.451348598946611, + 2.4318359998552737, + 2.4113861653214377, + 2.4102256642578697, + 2.4206051127750996, + 2.403003543258472, + 2.406640380136869, + 2.427282634730049, + 2.440338312768585, + 2.455002833218848, + 2.4366918339410732, + 2.450686023377833, + 2.392474597048947, + 2.43258917073127, + 2.4511340561491366, + 2.3878513471325853, + 2.423730652506104, + 2.454427435168003, + 2.399077576235631, + 2.4345817295184764, + 2.426571214495702, + 2.346396952134622, + 2.41716560900369, + 2.4535861781485213, + 2.4300612223873035, + 2.393237413405604, + 2.42276456500755, + 2.465298675382182, + 2.4184734534290913, + 2.4260860791846364, + 2.394397115953482, + 2.4324962616235233, + 2.4479096388674595, + 2.39759388888523, + 2.3428673360148577, + 2.39499526224599, + 2.4387494694799097, + 2.4144161777564905, + 2.4299147616029773, + 2.3856952697015075, + 2.386567526588384, + 2.3794113919123063, + 2.43288813400207, + 2.455924648573178, + 2.3940619800688916, + 2.4445137093377607, + 2.4769713699670017 + ], + "dft_magnitude": [ + 142.9001678991595, + 10.886321603643275, + 7.871503196858476, + 3.9177574724213757, + 0.234573180442305, + 3.104844509362704, + 4.728983130787377, + 5.0022770634387905, + 4.073811459123263, + 3.2139068502581774, + 1.9498026164567377, + 1.0023842845247208, + 0.23048645960665803, + 0.7411827663105564, + 1.8461520006854393, + 2.198199648689469, + 1.8898677611312744, + 1.8355749301360875, + 1.3220854190939675, + 0.8474439300709706, + 0.2958729186890929, + 0.726286102499572, + 1.4943750515858447, + 1.8042965878959878, + 2.087437779787597, + 1.319225583689189, + 0.8820253369130494, + 0.2456809800349068, + 0.18195279942957862, + 0.35116015280877744, + 0.9905360199483837, + 1.4796582016312065, + 1.5910472270400504 + ], + "mean_band_dot": [ + -4.666923901197606e-05, + -2.6972107264100487e-05, + -3.2159931038222567e-06, + 1.1411218281409674e-07, + -7.110654081543544e-06, + 1.0816063024776668e-05, + 5.923563207943516e-06, + 5.394922959567339e-05, + -4.8594380814392935e-05, + -2.4756280140536546e-05, + -1.3031123501150432e-05, + -1.115945451601874e-06, + -6.1339118246905855e-06, + 4.658945385926927e-06, + 2.308820427288083e-06, + 5.172361019845084e-05, + 4.5033001356387103e-05, + 6.25561302740607e-05, + 4.977796541183466e-05, + 5.3922856096733085e-05, + 5.882255959477334e-05, + 5.758099413810669e-05, + 4.126149109140442e-05, + 5.100415779679679e-05, + 5.6731379913799174e-05, + 5.6179472409212394e-05, + 5.726548548068422e-05, + 5.054369623280763e-05, + 5.2072665596369916e-05, + 5.444426500389454e-05, + 5.4958010309746896e-05, + 4.834910812689941e-05, + 5.6473469555839984e-05, + 5.629049505273542e-05, + 7.465503307457766e-05, + 5.273224162749557e-05, + 5.9408049267517526e-05, + 5.72085253907062e-05, + 4.872709666869923e-05, + 5.143241287441924e-05, + 5.957609418771881e-05, + 5.9924756953932956e-05, + 4.812480321447765e-05, + 6.662459943385102e-05, + 5.38371109541913e-05, + 4.145628747664887e-05, + 4.662309149239263e-05, + 5.920587119589982e-05, + 5.795741763847673e-05, + 5.4634360452610053e-05, + 4.961890454069362e-05, + 4.6977135269798964e-05, + 5.323800348833174e-05, + 6.176959323056508e-05, + 5.92820917120207e-05, + 5.696301562352346e-05, + 6.0797070602802705e-05, + 5.872998880818159e-05, + 6.61124616740949e-05, + 5.481329390022438e-05, + 5.94176211308195e-05, + 4.412055680802496e-05, + 6.270112905326641e-05, + 6.111529020813578e-05 + ] + }, + { + "layer": 10, + "mean_snr": 0.9041604968413336, + "per_head_snr": [ + 0.7973400904451962, + 0.9654719897203221, + 0.9813541490138673, + 0.8678695632082007, + 0.8384274306982469, + 0.9186718750661163, + 0.913292798259512, + 0.9747459840909684, + 0.8581121656442562, + 0.8401856785782176, + 0.905686108730541, + 0.9116822127369761, + 0.9339649485589631, + 0.8928419449089775, + 0.9690908879356288, + 0.8978301218653484 + ], + "mean_band_energy": [ + 0.9910653739375961, + 1.2109762336401153, + 1.3947311791349986, + 1.5778358393582763, + 1.861077371211826, + 2.054638226637457, + 2.4147625628646057, + 2.561031632533857, + 1.0152498568830737, + 1.1813736962965558, + 1.4666663275458756, + 1.6370468349823133, + 1.7593502940178944, + 2.18289503162329, + 2.2944725385833307, + 2.5476047909412705, + 2.584948848467449, + 2.55624649383752, + 2.5571959073267774, + 2.5581116628165432, + 2.5978812177054085, + 2.6044936247358965, + 2.548808577028499, + 2.5414765556448042, + 2.5613113825559735, + 2.615447117451337, + 2.5722260838205075, + 2.5693189820004267, + 2.5783079018526402, + 2.61572761228393, + 2.5877538424111197, + 2.5929205872493695, + 2.514527915096034, + 2.533655440593906, + 2.5839733382876853, + 2.6171737879540693, + 2.584421652332888, + 2.563411543716743, + 2.5454377754284945, + 2.59014072521624, + 2.6056254846121245, + 2.5876583911077615, + 2.5835316177861802, + 2.622965036360683, + 2.6212731975033483, + 2.592328609729999, + 2.5209242931681928, + 2.5920289187160046, + 2.5727909762786165, + 2.5870089196278947, + 2.5824462850074914, + 2.5561733034623195, + 2.550608647276631, + 2.568272708188104, + 2.5994578848131873, + 2.608361696032393, + 2.5464093547424564, + 2.5458032781277753, + 2.6203663734899827, + 2.600584338804585, + 2.5460593758964594, + 2.571152645366766, + 2.5786849372027154, + 2.5577360481753937 + ], + "dft_magnitude": [ + 151.84394868748367, + 11.998019659170867, + 8.582256637451138, + 4.307125622090811, + 0.3329099732102131, + 3.525195707577094, + 5.310007680462848, + 5.544107628980873, + 4.74736368738185, + 3.5363047792491864, + 2.170124658015844, + 0.9409976374350734, + 0.045413759745338005, + 0.7585455059070974, + 2.303372006961719, + 2.4790040380030054, + 2.634244363329681, + 2.4096191453781657, + 1.5309573151331026, + 0.7211217091023581, + 0.24708859847112477, + 1.0448093071869753, + 1.557738357404905, + 1.7804869831638808, + 1.9020009791315202, + 1.4855540183114295, + 1.0553105332699317, + 0.8001619918875232, + 0.4290299898077242, + 0.43279377566126753, + 0.9999286781014265, + 1.3710843576506364, + 1.959251938943538 + ], + "mean_band_dot": [ + -4.4124136309164896e-05, + -2.091436181217432e-05, + -1.261510563210777e-05, + -1.2026265750364473e-05, + -1.6560094380224655e-05, + 2.1343936225548532e-05, + 3.1229878061367344e-05, + 9.475013089854656e-05, + -4.005608024115759e-05, + -2.294673888059151e-05, + -1.460614426207485e-05, + -6.737167922210574e-06, + -5.578359235869357e-06, + 5.818929324163946e-06, + 1.4912061871541482e-05, + 9.565118736531986e-05, + 0.00011328599259741168, + 0.00010596555162578625, + 0.000103878442473615, + 0.00011083999713434878, + 0.00011490570358319019, + 0.0001051579164368377, + 0.00010561961386201801, + 9.950062093366796e-05, + 0.00010146846256020581, + 0.00011130173766105145, + 0.00010583330596602991, + 9.739362837990484e-05, + 0.00010391952395139015, + 0.00011561959810535427, + 0.00010101891703584442, + 0.00010199302721503045, + 9.019300162549371e-05, + 9.328677069220248e-05, + 0.00010267474851843872, + 0.00010458925197553981, + 0.00010441421264317796, + 9.564479564971862e-05, + 0.00010485501599077907, + 0.00010390343847177519, + 0.00010835232359340807, + 0.00010769487414563628, + 0.00010098938540181734, + 9.599188655329272e-05, + 0.00010508550390397886, + 0.00010051282890799484, + 9.204556147324183e-05, + 0.00011011119704562589, + 0.00010711099497484611, + 0.00011313525854461659, + 0.00010317410078641842, + 0.00010101003573481648, + 0.00010374745545504993, + 0.0001034120974736652, + 0.0001175024191581997, + 0.00011072841355996843, + 9.843372430395902e-05, + 0.00010502032620252066, + 0.00011329886274324963, + 0.00010903643484994063, + 9.026847988025111e-05, + 0.00010597928263678114, + 0.0001169483380678571, + 9.523743817396735e-05 + ] + }, + { + "layer": 11, + "mean_snr": 0.8974714098367496, + "per_head_snr": [ + 0.8990677713942579, + 0.9179604556819306, + 0.9014961875689552, + 0.9200885758861589, + 0.9378835512527447, + 0.8730790483451106, + 0.8561765285521077, + 0.9470702219111373, + 0.8339551517436282, + 0.9466037094270259, + 0.8688437371465552, + 0.8826835910707097, + 0.8740528170512726, + 0.8784880893052524, + 0.896517803209136, + 0.9255753178420105 + ], + "mean_band_energy": [ + 1.1822798460479425, + 1.2856421835001088, + 1.5362262193960887, + 1.7070683241669569, + 2.014651081245529, + 2.2252136447340423, + 2.39366554192, + 2.575817003699081, + 1.079624298401598, + 1.2652909616771926, + 1.4848312786265772, + 1.7667743501962234, + 1.9446460472127098, + 2.1893284456236817, + 2.4893279453804946, + 2.592362854632084, + 2.7176580607367278, + 2.76960956076584, + 2.7686366401686984, + 2.7151228969491683, + 2.6593136648759703, + 2.729451421434704, + 2.718264303241619, + 2.7532500570050313, + 2.7629155314470957, + 2.713377292339386, + 2.7684200382271165, + 2.6764833842969136, + 2.7142807244261338, + 2.6542504341466238, + 2.7367951528322214, + 2.6770304618622296, + 2.683851452626632, + 2.6911041757072027, + 2.6984941104960525, + 2.7418709357849655, + 2.6742686355481435, + 2.6914563483444613, + 2.7599202714373092, + 2.742572807280193, + 2.7656572766417655, + 2.760609195968957, + 2.652763419196135, + 2.713380630660586, + 2.6921591372825997, + 2.7434563773376768, + 2.7389701788752445, + 2.7406849628330283, + 2.704297422050285, + 2.7292745298161254, + 2.7023493285325904, + 2.6765897639193454, + 2.7608153042405945, + 2.6679972562100502, + 2.748667766552159, + 2.7193452919763548, + 2.7224710545188726, + 2.685352416665163, + 2.780275817004787, + 2.6979874848334795, + 2.711522930301908, + 2.731456389344971, + 2.6006492389867857, + 2.7129176445368213 + ], + "dft_magnitude": [ + 160.11079920672705, + 12.491601075973493, + 9.241391946324605, + 4.731819016721624, + 0.15431680291147815, + 3.2617500730317137, + 5.3973387097163945, + 5.387209069835327, + 4.556386788331324, + 3.453213377260745, + 2.3147410105026247, + 1.5155444593294793, + 0.12391149428907328, + 0.8798733625045588, + 1.7760988997821094, + 2.072240893009372, + 2.450850145689371, + 2.065694750869474, + 1.461557478342206, + 0.5593642420821139, + 0.11271125357053727, + 0.6348510521754056, + 0.9136839263248182, + 1.849485974367115, + 1.7659422939593563, + 1.736340156193418, + 1.4850484295287585, + 1.1712592101957648, + 0.08525829898406348, + 0.7772189354708577, + 1.3354612318401289, + 1.4658707716949932, + 1.373459769770264 + ], + "mean_band_dot": [ + -3.069707787517473e-05, + -2.583893792973413e-05, + -8.706847552275576e-06, + -1.021661756794856e-05, + -1.1334070876500846e-05, + 1.3471615574189855e-05, + 2.9963098256757803e-05, + 8.389679470610645e-05, + -3.0817298103613666e-05, + -1.5877140015163604e-05, + -1.0401434138884724e-05, + -1.9010261290475228e-05, + -8.007587325664645e-06, + 1.895043482136316e-05, + 2.530815947920928e-05, + 8.569090685739411e-05, + 0.0001004394716517254, + 9.324497898433037e-05, + 9.973649946459773e-05, + 9.047467858636082e-05, + 9.652109559965538e-05, + 9.184626949831909e-05, + 8.367448287671664e-05, + 9.673638541585206e-05, + 9.526709993679106e-05, + 9.223258123824962e-05, + 9.870554515600816e-05, + 8.85404958808067e-05, + 9.791555618221537e-05, + 9.49932814364729e-05, + 8.744862387288777e-05, + 9.147934867215213e-05, + 9.654070229458966e-05, + 9.189015938204648e-05, + 9.736347524835764e-05, + 9.516894476746529e-05, + 9.641520294678685e-05, + 8.677281738300735e-05, + 9.585482985485783e-05, + 9.708531214869254e-05, + 8.979796984931454e-05, + 9.454210865555979e-05, + 9.648042293974868e-05, + 8.772139364054965e-05, + 9.560508777894938e-05, + 0.00010501544078067582, + 9.53066642352951e-05, + 0.00010173097223287186, + 8.976116420456037e-05, + 9.31400740000754e-05, + 8.490354113632748e-05, + 9.602765180716233e-05, + 0.00010678881312742305, + 9.211001813014263e-05, + 9.63537315215035e-05, + 9.366323416770683e-05, + 9.123950070488718e-05, + 9.380247240642348e-05, + 0.00010781068385767867, + 0.00010236257219276013, + 9.584757901848207e-05, + 9.944244970938598e-05, + 8.638993779186421e-05, + 9.661754143053258e-05 + ] + }, + { + "layer": 12, + "mean_snr": 0.9118770789847048, + "per_head_snr": [ + 0.9099314255353699, + 0.9269460809781692, + 0.9188882005980273, + 0.9798949997112121, + 0.8551106558626927, + 0.9523311587826282, + 0.8843794787725519, + 0.9200029777163514, + 0.9834282810711196, + 0.9258035822452232, + 0.8495838594677625, + 0.9093409309223183, + 0.9339310888603809, + 0.8585935323954585, + 0.8685233968622544, + 0.9133436139737585 + ], + "mean_band_energy": [ + 1.3004197806411846, + 1.4974981496907782, + 1.730603416446594, + 1.958059126725196, + 2.0775054022570076, + 2.404031465724268, + 2.4868473542558798, + 2.5760207958937933, + 1.3372425937574235, + 1.4396313328788652, + 1.6894239026945783, + 1.7864228540233578, + 2.133534507094427, + 2.227055454740361, + 2.3684719224211332, + 2.568742235494022, + 2.7440393087038126, + 2.789983491616736, + 2.8090728425590106, + 2.745810617574384, + 2.744443678500466, + 2.720955455623338, + 2.701945921819256, + 2.7405848689395853, + 2.7605496755281935, + 2.6477195642690035, + 2.687331623064842, + 2.7836100678961007, + 2.7209590159688393, + 2.7105844147869984, + 2.727336788134563, + 2.6691695712443337, + 2.7326800746916042, + 2.649428428829909, + 2.742693737696313, + 2.715915166939517, + 2.763180386045452, + 2.744207754146558, + 2.7318796780264547, + 2.7713421222378587, + 2.786214260753301, + 2.6961807513792078, + 2.7532370750701816, + 2.776303136035555, + 2.746341999286316, + 2.7495453162114396, + 2.769949549069005, + 2.640717539605519, + 2.6751418219409, + 2.7417208185648887, + 2.713579211775931, + 2.707556580736221, + 2.7349376426303182, + 2.7249755834130394, + 2.796240188486192, + 2.738195890231994, + 2.663955985090408, + 2.737416290044524, + 2.775977091786488, + 2.740099991755624, + 2.7560460292642857, + 2.728734493723022, + 2.7279239863423976, + 2.7467120750393503 + ], + "dft_magnitude": [ + 162.76463785781812, + 10.826479027049317, + 7.76318084648854, + 4.253741383879098, + 0.5005306804090603, + 3.3558736593754843, + 4.766108948920928, + 4.658791615311151, + 4.127516302512609, + 2.9610543452955795, + 2.312527029209647, + 1.324308250784158, + 0.4365829164394839, + 0.938102955465051, + 1.6985989681540228, + 2.3190657715444707, + 2.1156482607762106, + 1.5159474967662065, + 1.4709859826435, + 0.9632918598301363, + 0.19208859376034423, + 0.10381583099779632, + 1.1104123150731628, + 1.9534320974880628, + 1.4747099416475, + 1.353355974188699, + 0.872368349692907, + 0.2731591067796265, + 0.22729716365965216, + 0.9375826469199363, + 1.095677886742081, + 1.5316387228468558, + 0.9852249542125833 + ], + "mean_band_dot": [ + -2.3541833513718302e-06, + 6.726564926395895e-06, + 2.669854177383968e-05, + 2.3658447929619797e-05, + 2.5320172980514144e-05, + 3.5190667745155224e-05, + 2.5420200643111457e-05, + 6.883579550276409e-05, + -9.707444718287661e-06, + 8.875860629586896e-06, + 1.319234021224247e-05, + 3.279639865638728e-05, + 2.403074887524781e-05, + 3.6288658066041535e-05, + 1.7898868733823292e-05, + 6.978122257805808e-05, + 6.991696166380734e-05, + 7.785102638990793e-05, + 6.96494745398013e-05, + 7.681819164417902e-05, + 7.770546505980747e-05, + 7.395751676142481e-05, + 7.422133569434664e-05, + 7.37310519696166e-05, + 7.287535157729508e-05, + 7.18436130568989e-05, + 7.444846164617047e-05, + 7.91732916241017e-05, + 7.530845263659103e-05, + 7.440288703719489e-05, + 7.943659261400171e-05, + 7.685827011982838e-05, + 7.350058237420853e-05, + 6.543643664258525e-05, + 7.665136905643521e-05, + 7.711842545177205e-05, + 7.007489602983696e-05, + 7.073559493164794e-05, + 7.515421393122779e-05, + 7.846542058587147e-05, + 7.559422959957374e-05, + 5.97522184762056e-05, + 6.84660313936547e-05, + 7.787289150940069e-05, + 7.652853577155838e-05, + 6.708015797585176e-05, + 6.847344104699005e-05, + 5.911674679737189e-05, + 7.572875290406955e-05, + 7.232588249905803e-05, + 7.348059212830549e-05, + 6.345621602577012e-05, + 7.127876423851376e-05, + 7.098348987710779e-05, + 7.283996990281594e-05, + 7.183639570484956e-05, + 7.493086394561033e-05, + 6.51200350603176e-05, + 6.659580014911626e-05, + 7.961027080227723e-05, + 7.271142726494872e-05, + 7.522635343093498e-05, + 7.232463173068027e-05, + 7.509658212256909e-05 + ] + }, + { + "layer": 13, + "mean_snr": 0.8797259055605411, + "per_head_snr": [ + 0.8380164180726992, + 0.7963446351423371, + 0.8834013841636685, + 0.8534197663069566, + 0.9481501923274773, + 0.9097337761494018, + 0.8574783976291649, + 0.8096924037445052, + 0.9296251040892909, + 0.9679660636678923, + 0.8297356858699448, + 0.9287050214802589, + 0.9121847633540862, + 0.8865889955560904, + 0.8830690231987325, + 0.8415028582161517 + ], + "mean_band_energy": [ + 1.5228725213699592, + 1.6779019378875328, + 1.7700127881235308, + 2.0674154481995455, + 2.2470353486345838, + 2.8072361681173152, + 2.818916177693527, + 3.0972325360442845, + 1.5080492047877576, + 1.6330647169581813, + 2.016148120815383, + 1.9559446206782036, + 2.276154694635448, + 2.5630562244801913, + 2.7886522559761584, + 3.020093831380393, + 3.5380842526355343, + 3.4273819077947225, + 3.487790999892699, + 3.3862024223977514, + 3.386038516955592, + 3.2717743734989275, + 3.379824075156087, + 3.503406716503921, + 3.514991107825848, + 3.4377730665299557, + 3.43866606152922, + 3.3988596787808465, + 3.4648722025216143, + 3.390815247576767, + 3.5414656480690523, + 3.371057959615111, + 3.4514898569030557, + 3.4198091335632967, + 3.40514108817435, + 3.4548479180856866, + 3.515668658669723, + 3.4701499708270322, + 3.511357978132314, + 3.5496409603288095, + 3.4753788330194624, + 3.4435343109598575, + 3.5948639653780514, + 3.434455419463891, + 3.484095588623883, + 3.460334649845179, + 3.4690190218556625, + 3.468523551240145, + 3.5637323693174805, + 3.5469529808736455, + 3.4605493774892366, + 3.5587830665600633, + 3.55813560552045, + 3.48983430822764, + 3.4892832459002, + 3.5009802614509002, + 3.547512709282513, + 3.4023928893191435, + 3.415171946858343, + 3.4179555545034743, + 3.464603193301661, + 3.385860769717792, + 3.5002262055870963, + 3.534754550877399 + ], + "dft_magnitude": [ + 202.15383077292307, + 18.089577981453655, + 12.34721797038512, + 5.771267028434835, + 0.39494321279002015, + 3.873695629175757, + 6.782555758387039, + 6.49095696908064, + 4.9785859104825265, + 3.6937111877966498, + 3.735450637007304, + 1.9023944366001484, + 0.19941085178643134, + 1.096278330240547, + 2.000284831193649, + 2.2928888944874535, + 2.457748661582856, + 2.309348237079336, + 2.414077381187276, + 1.36779795564373, + 0.5801834404847585, + 1.041545868098582, + 2.3345615257794363, + 2.0934445487657536, + 1.963132389984591, + 1.4713533828546475, + 0.8352331184742023, + 0.8449921332226503, + 0.9152441692105614, + 1.4608082087891305, + 1.9004859547216144, + 2.240748388093396, + 0.9422235316521324 + ], + "mean_band_dot": [ + 9.727549013405223e-06, + 1.2214917092023825e-05, + 3.987612984701627e-06, + 1.5542744606023007e-05, + 2.0920255877854288e-05, + 3.247452082177915e-05, + 2.552763490371035e-05, + 9.888690073012185e-05, + 7.2696448398801294e-06, + 1.3111669858290043e-05, + 2.243507813659562e-05, + 1.1063951447454201e-05, + 1.2917290064251574e-05, + 3.210506454820461e-05, + 2.7273808896666196e-05, + 9.961849855244509e-05, + 0.00010261495185659442, + 0.00010362839088884357, + 9.832007106069796e-05, + 9.35562931658751e-05, + 0.00010711580603128823, + 9.781244079931639e-05, + 8.918617547237773e-05, + 9.891840460340973e-05, + 9.696362371869327e-05, + 0.00010497615357962786, + 0.00010236697733034816, + 9.63658691262026e-05, + 0.00010405163243376592, + 0.00010618429598707735, + 0.00011330807137710508, + 8.8205030692734e-05, + 0.00010066363392979838, + 0.00010211218886979623, + 0.0001037229001212836, + 9.511676967122185e-05, + 9.836813671881828e-05, + 0.00010039382914328598, + 9.79648569341407e-05, + 9.464683387250261e-05, + 9.401114834872715e-05, + 9.278412659341484e-05, + 0.00010353377683713916, + 9.746816090228094e-05, + 9.693522343923178e-05, + 0.0001044729579007253, + 9.978136063182319e-05, + 0.00010119621538251522, + 0.00011466091086731467, + 0.00011540521609276766, + 8.990001219899568e-05, + 9.826418585134888e-05, + 0.00010198269524153147, + 9.932055127137573e-05, + 0.00010397994356026175, + 0.00011441741139606165, + 9.275910521466812e-05, + 0.00010606893147269147, + 0.00010513174356674426, + 9.708506300398767e-05, + 0.00010053499784135056, + 0.00010027611131135927, + 9.969463894776709e-05, + 0.00010795386310746835 + ] + }, + { + "layer": 14, + "mean_snr": 0.8861099649472839, + "per_head_snr": [ + 0.8201592121982804, + 0.9312669336258719, + 0.8950825102832626, + 0.9626945165574553, + 0.8676428009786319, + 0.922042784417827, + 0.8466726573233467, + 0.8817892405132465, + 0.9291933194759355, + 0.9494060546410849, + 0.8752885263354163, + 0.823145205948269, + 0.8358188003312305, + 0.887056573011581, + 0.8611951918523433, + 0.8893051116627593 + ], + "mean_band_energy": [ + 1.6926169061514313, + 1.7658658000452991, + 1.9171588308815215, + 2.064636245250986, + 2.412621416609694, + 2.9724706890641297, + 3.052206196567907, + 3.364040126519769, + 1.576655661688461, + 1.693492829937942, + 2.0191202369402586, + 2.1049782416099116, + 2.5539661168213703, + 2.8043443841873854, + 3.1038891299334415, + 3.251513814266686, + 3.907125003270755, + 3.691543988205356, + 3.8506795088910772, + 3.780312241013129, + 3.659955485995111, + 3.714893255594416, + 3.81440938444695, + 3.700120358124183, + 3.678231326150943, + 3.676346184349968, + 3.7829844252035736, + 3.8410205221778106, + 3.707587519290705, + 3.7460399511661784, + 3.7441073557297244, + 3.811247967585069, + 3.6357109215139074, + 3.8320993408186768, + 3.6729956541834365, + 3.831457416459121, + 3.603160587779991, + 3.7808970335115366, + 3.729802554284511, + 3.7520150612047907, + 3.768625099748105, + 3.810308394643865, + 3.7598308386467822, + 3.7098863244089273, + 3.7543906772129105, + 3.6029346753495917, + 3.7401805942012576, + 3.68181525447877, + 3.7157281246414486, + 3.7885879473719193, + 3.6426493240272393, + 3.6363652020441943, + 3.7738070404201096, + 3.7518773264383922, + 3.6901001543628458, + 3.678535568455131, + 3.6940506472141577, + 3.8017301434663597, + 3.7555582870364335, + 3.7079960354551167, + 3.80360819129693, + 3.79184010923478, + 3.8077415993364117, + 3.719893651827457 + ], + "dft_magnitude": [ + 217.88236088474625, + 19.156584532194792, + 13.898394973212483, + 7.435322902470171, + 0.3668217696668279, + 4.891542774266957, + 7.573759844390122, + 7.188523751636731, + 5.5825383186468756, + 4.776998600605345, + 3.1092878580540413, + 2.149551407351517, + 0.6673055322969584, + 1.7669082353066146, + 2.663381047216814, + 2.351570566889666, + 2.567716901787492, + 1.8319567111216661, + 1.9469981182491394, + 1.312239678915765, + 0.0961362631925851, + 1.2992399672213721, + 2.0294636924960217, + 2.8867967041157914, + 1.91109823224676, + 1.5677470326465366, + 1.444164208148176, + 1.0752513327692557, + 0.8929432656071583, + 1.0417349005541818, + 2.6610147148523033, + 1.1265923481538975, + 1.8398512837874534 + ], + "mean_band_dot": [ + 2.025754125156709e-05, + 2.3242595602823712e-05, + 3.072370884638076e-05, + 2.2776888201292422e-05, + 1.1109015304100467e-05, + 4.0842344390057406e-05, + 4.7355917558888905e-05, + 0.00011630202658352573, + 1.8768675971614357e-05, + 1.8920901290186976e-05, + 2.2623153938639007e-05, + 1.3700138538297324e-05, + 1.6698484117227963e-05, + 4.9025158780580114e-05, + 3.990855719848696e-05, + 0.00011674529059746419, + 0.0001229299811029705, + 0.00011660643565392093, + 0.0001267652686465226, + 0.00013258397757454077, + 0.00011145423684411071, + 0.00010406160242837359, + 0.00011803103089391698, + 0.000127678199305592, + 0.00010481650423344036, + 0.00011154470232099811, + 0.00011858410775289485, + 0.0001228854775945365, + 0.00012001186462384794, + 0.00013069649781982662, + 0.00011579977302744737, + 0.00012955011482063128, + 0.00011693175275695467, + 0.00012039406954045262, + 0.00011933165035316051, + 0.00011296791899439995, + 0.00010846133413622283, + 0.00012177021790193976, + 0.00012027720333662728, + 0.0001296882390988685, + 0.00012271032777277924, + 0.00013017584171848284, + 0.00012455461569516046, + 0.0001122092550644993, + 0.00012700646288976714, + 9.539616395670691e-05, + 0.00012382813963540684, + 0.00011872134274426571, + 0.00010363284974346243, + 0.00012520983185027035, + 0.00010713657468386373, + 0.00010786337162471682, + 0.00011680281382098201, + 0.0001306847323121474, + 0.0001288379859829547, + 0.0001059790685644657, + 0.0001264444878188442, + 0.0001221959382746718, + 0.00012498682053774246, + 0.00012554041518342274, + 0.00012267470799542934, + 0.00012097072465167003, + 0.00011665343242839299, + 0.00012984025624973583 + ] + }, + { + "layer": 15, + "mean_snr": 0.8641094254012319, + "per_head_snr": [ + 0.82618224493334, + 0.8206804575670412, + 0.8226451066598991, + 0.839370173816753, + 0.8514226631339631, + 0.7922496473580998, + 0.901816943839525, + 0.9061162733854092, + 0.8834570509005013, + 0.8408632102691552, + 1.038061671531925, + 0.8271807162476653, + 0.8905706906669063, + 0.8938057762282213, + 0.8399437128726153, + 0.8513844670086891 + ], + "mean_band_energy": [ + 1.6293416596834172, + 1.6917000914281668, + 1.9170749408926797, + 1.9233835205514733, + 2.370825453802099, + 2.9463913539877042, + 3.089863785077001, + 3.586159154944486, + 1.5863294949614595, + 1.7352698129033874, + 1.8869536636092183, + 2.0756032014915746, + 2.4591869396855524, + 2.7459118951991632, + 3.069625104382256, + 3.430114280726168, + 4.1004801919054605, + 3.961580175193692, + 4.145258137581976, + 4.222333405614965, + 4.062403631121974, + 4.117193730892001, + 4.130158762521084, + 4.166271901454996, + 4.14525960593358, + 4.1849437085845995, + 3.991569559413624, + 4.0598419954138745, + 4.07851542820357, + 4.087108715249748, + 4.056642653713539, + 3.9985803053015196, + 4.052733142191319, + 3.94133454609599, + 4.07787513416713, + 4.099822983997944, + 3.990377003388595, + 4.05829856268473, + 4.077393570938933, + 4.005643195050151, + 4.121016225952494, + 4.044764414306348, + 4.147912073103296, + 4.051014932422986, + 3.9126200455138136, + 4.168119014215351, + 3.963029134886681, + 4.164923452485606, + 4.077491224033577, + 4.109096776214299, + 4.1033179178701396, + 4.013038604876324, + 4.170636434614386, + 4.108078317463853, + 4.0466973802376405, + 4.046420585949026, + 3.996384346538118, + 4.134839885888699, + 4.092334416866373, + 3.9471303357413534, + 4.107694842432817, + 4.047714655757119, + 4.0642720467999345, + 4.133522254732017 + ], + "dft_magnitude": [ + 233.72742371884306, + 24.11354975940974, + 18.064860262248363, + 8.302862550807305, + 0.6486786540771939, + 5.8955370180549105, + 8.621501110077418, + 8.009423025790909, + 6.18275079602008, + 4.812651669685135, + 3.650717520713613, + 2.390685731804668, + 0.23607134484904, + 1.6688886820576303, + 3.020056906475452, + 3.3224629165860455, + 2.717664700483825, + 1.997833917206094, + 2.2201395696693647, + 1.2983324736690824, + 0.6268667734070468, + 1.2865826546133599, + 2.691981348048575, + 3.0016057026719, + 2.6506074859061894, + 3.1063117454759226, + 1.184592113186661, + 1.548622647454582, + 0.43470855560863436, + 1.9592610884968897, + 1.6023530473647998, + 2.0009773896124736, + 2.2848758147955834 + ], + "mean_band_dot": [ + 2.164475691301959e-05, + 1.4207230975671337e-05, + 1.193390519915738e-05, + 3.980052682095447e-06, + 3.186145569600285e-05, + 5.508092931449937e-05, + 5.730857685648516e-05, + 0.00012644426874430792, + 2.7912551189501755e-05, + 2.1372007125819437e-05, + 1.7251303859211475e-05, + 1.6101949348978906e-05, + 9.434840393396371e-06, + 5.454705046759045e-05, + 5.8587207803384445e-05, + 0.00011331768422451205, + 0.00012829796787627856, + 0.00012917027220282762, + 0.00013712669806409394, + 0.0001399423736074823, + 0.00014586126826543477, + 0.00014407000742266973, + 0.0001490873769398604, + 0.00014568353765298525, + 0.00013971083535579965, + 0.00015023147301462814, + 0.00014264803644437052, + 0.0001448517357403034, + 0.00013912083886680193, + 0.00014707380694289895, + 0.000132946766598252, + 0.0001373951281493646, + 0.00013295851988459617, + 0.00011904631708148372, + 0.00013413429843467384, + 0.00014611374513151532, + 0.00015404111559291778, + 0.0001341971230033323, + 0.00014767511584068416, + 0.00014360701857185632, + 0.0001398182821503724, + 0.00014195691278473532, + 0.00014518290117848665, + 0.00013166432472644374, + 0.00013870553061678947, + 0.00014231937723252486, + 0.00014251515608521004, + 0.00013794357050755934, + 0.00014709842366755765, + 0.00014118638819127227, + 0.0001305116009007179, + 0.00013268844168123906, + 0.00013419875961062644, + 0.00014757706713908192, + 0.0001369863796298887, + 0.0001353921209101827, + 0.00013760079218627652, + 0.00013264243420962885, + 0.00014416705334951985, + 0.00014083691132782405, + 0.00014514957516098548, + 0.00013167667756874835, + 0.00013227796000592207, + 0.00013867860184291203 + ] + }, + { + "layer": 16, + "mean_snr": 0.8673081418694217, + "per_head_snr": [ + 0.8401607778243126, + 0.8401557451558999, + 0.8384538195127732, + 0.7618391422182028, + 0.8235637586651011, + 0.8246534222199223, + 0.8736678863864945, + 0.8640085482549245, + 1.119920561364215, + 0.8336667303908937, + 0.9093585348241429, + 0.8680993156965802, + 0.946473151094582, + 0.8132511858315985, + 0.8591486565779384, + 0.8605090338931649 + ], + "mean_band_energy": [ + 2.0877519415404038, + 2.1086881052983566, + 2.1625988571773775, + 2.1944864779365765, + 2.6823490633595073, + 3.235266786671211, + 3.2848689087332135, + 3.835433452414735, + 2.027507356210067, + 2.155387752683743, + 2.282802897074383, + 2.413309579311969, + 2.5796517647665587, + 3.2549971826273447, + 3.189664924061136, + 3.936765895483237, + 4.400276117532241, + 4.496637794493164, + 4.393138245202838, + 4.520092838480018, + 4.543408934545123, + 4.500075835537961, + 4.120781545848042, + 4.501493291798964, + 4.443533998121882, + 4.444768390791106, + 4.577839480958697, + 4.467302148030594, + 4.262275171879734, + 4.453106083735446, + 4.36985941906357, + 4.401143108928165, + 4.4840050581885365, + 4.423282316922208, + 4.535229255239587, + 4.431064604351471, + 4.511436165802409, + 4.402116317423747, + 4.51069934269805, + 4.5810979200584665, + 4.5781353094745985, + 4.530190201371189, + 4.428840867201043, + 4.473505395632003, + 4.555414186840046, + 4.233565688862104, + 4.501587665942023, + 4.528322032586472, + 4.567822891684273, + 4.4803645502116884, + 4.52250481115961, + 4.599624216022278, + 4.583893241994598, + 4.493471537941479, + 4.421234409352458, + 4.488107074587866, + 4.4940614681521405, + 4.479390861449684, + 4.584282822523406, + 4.519060985933637, + 4.525200817297094, + 4.628481721881005, + 4.506953857429618, + 4.586872304773973 + ], + "dft_magnitude": [ + 258.5170832512861, + 25.37088124844629, + 18.245133361396636, + 9.496039615377827, + 0.16099314480490456, + 5.6521836976191, + 9.48194886695752, + 8.350822154128243, + 5.193688932235832, + 4.891246031287916, + 3.34720058955524, + 1.992496452976777, + 0.8733694920569298, + 1.1697870272653959, + 3.194913336434622, + 2.312824085528905, + 2.4069057738943007, + 2.4289457884140213, + 2.0511486053900754, + 1.7059638118637954, + 0.6491904478597942, + 1.825456674361262, + 1.5986267483495264, + 3.44399259816988, + 3.0693701588388067, + 1.357229754531442, + 2.1640047384551986, + 1.099152366126963, + 0.19600796090179168, + 0.737411586719786, + 1.1889713275242475, + 3.145164240564496, + 3.0778616571775927 + ], + "mean_band_dot": [ + 6.3493556837102e-05, + 5.058858990736326e-05, + 2.1181176848017458e-05, + 2.1575300621634597e-05, + 2.666190619038389e-05, + 7.236109485120323e-05, + 5.878260014924308e-05, + 0.0001627978244869155, + 6.530848335373207e-05, + 4.3119306397443324e-05, + 3.645796812179469e-05, + 3.2469341839203025e-05, + 3.940251036738118e-05, + 7.162426871332173e-05, + 6.0643442566288286e-05, + 0.00016786510286692646, + 0.0001587683590571487, + 0.00017174880520087754, + 0.00016840957897557018, + 0.00017082966081716222, + 0.0001625138546614835, + 0.00016964405179464848, + 0.00013597811602039656, + 0.00016090408631441733, + 0.00016570784259783977, + 0.00016709240278345305, + 0.00016764579618211428, + 0.00015678491263315664, + 0.00014281368351021229, + 0.00016777650085941787, + 0.00015916836380824861, + 0.00016391223741152317, + 0.00015742180664801708, + 0.00017911925675662133, + 0.00015008944291139414, + 0.00015943577784582885, + 0.00017006569362365553, + 0.000156691800924591, + 0.00015883888609380392, + 0.00016475235042889835, + 0.0001676787320548101, + 0.000167760737667777, + 0.00015282135649385964, + 0.00016643483633060896, + 0.0001698251378599025, + 0.0001458892545542767, + 0.00015189643079338566, + 0.0001608710401086455, + 0.0001678169330716628, + 0.00015689289524800643, + 0.00015708719047324848, + 0.0001646795458896122, + 0.00016249094664999575, + 0.00017203320328462723, + 0.0001707920561671017, + 0.00015962780997824666, + 0.0001656675181038736, + 0.00016174958568626607, + 0.0001604083784627619, + 0.0001689617796927223, + 0.00016093310745191047, + 0.0001583658776098673, + 0.00016493680843154834, + 0.00016837127964208776 + ] + }, + { + "layer": 17, + "mean_snr": 0.8863181113196326, + "per_head_snr": [ + 0.8334075133969867, + 0.8547551328748803, + 1.1090049083836568, + 0.8852479043105889, + 0.8616827134809527, + 0.8854978040009011, + 0.9256330907419569, + 0.825598404395169, + 0.8669643741578, + 0.9137459360290411, + 0.817162789421168, + 0.9639108170218957, + 0.8644125828451082, + 0.8297220342622994, + 0.8705912926181273, + 0.8737524831735881 + ], + "mean_band_energy": [ + 1.978542214499622, + 2.1008539854251067, + 2.1637992252776153, + 2.365395238839444, + 2.66157450865028, + 3.4264594302263234, + 3.4803182868672256, + 3.944296439380336, + 1.9662226084524794, + 1.9564689503273067, + 2.114327507452484, + 2.3298083786474035, + 2.7678509242905402, + 3.362064079371603, + 3.3246519554674894, + 3.8669036899431823, + 4.6129522539863475, + 4.667664970717098, + 4.534289771157603, + 4.461824128816096, + 4.399479546471039, + 4.622716637666878, + 4.54223262532461, + 4.5907794318753785, + 4.505639834009347, + 4.545459270906112, + 4.586177902444739, + 4.59686221207857, + 4.576911762614282, + 4.420506321169276, + 4.600587406440292, + 4.43077093797648, + 4.560908979897766, + 4.454773831714354, + 4.486759936463563, + 4.494297030007752, + 4.591433884477928, + 4.58742353481294, + 4.448034227647545, + 4.575562757954565, + 4.611974939713038, + 4.63043861443512, + 4.536381968492248, + 4.543758814819453, + 4.557048875839561, + 4.686111181869397, + 4.2177141812782235, + 4.55930949134952, + 4.593864397449151, + 4.40976198749377, + 4.5690233549416455, + 4.253259147582082, + 4.419602346937902, + 4.559444376026567, + 4.4634667116951245, + 4.510117530698217, + 4.674132139672928, + 4.672256881809016, + 4.502313818365721, + 4.592927430535292, + 4.573573608156179, + 4.441640816117917, + 4.386033111254933, + 4.572288213000312 + ], + "dft_magnitude": [ + 261.24003055928233, + 25.83186768352832, + 18.463785571020953, + 9.736618696526964, + 0.963021725310724, + 6.724876547472003, + 9.281898752373914, + 8.359595587804758, + 6.290925830819551, + 5.056498332850166, + 3.977864998146237, + 2.306646225770043, + 1.1578938265618595, + 2.27789619448863, + 3.4743642752149526, + 2.6208640721653347, + 1.458402989486264, + 2.7721254946428298, + 2.68142508029077, + 1.9971635353453783, + 0.6799162624591713, + 1.932518896770895, + 3.233873696257492, + 2.2387620111190896, + 2.8086167729718685, + 2.431046501737514, + 0.8674179354104935, + 0.9612539517098637, + 0.3517012075181895, + 1.8822870974758767, + 1.9788297463979179, + 2.84982967967587, + 3.224380927903411 + ], + "mean_band_dot": [ + 6.82892193282214e-05, + 6.83807597496866e-05, + 4.537633628132198e-05, + 3.8822788638981365e-05, + 4.317183523028234e-05, + 7.497745346540796e-05, + 6.837567843831494e-05, + 0.00012653262558615097, + 6.178170100845648e-05, + 5.359258067016981e-05, + 4.950099466327629e-05, + 3.804533382023578e-05, + 5.357038388353885e-05, + 6.173831798150786e-05, + 4.1614445791537946e-05, + 0.00013603547677121242, + 0.00013986951944389148, + 0.000142962541531233, + 0.0001579792509573963, + 0.00014138591814116808, + 0.00014219116309988067, + 0.00015362908078486726, + 0.000147752085808861, + 0.00014537721244778368, + 0.000144772991973241, + 0.00015328211543419457, + 0.00015777233943481406, + 0.00015095543579946025, + 0.00015279095117648467, + 0.00012674481511254498, + 0.0001412727802119207, + 0.0001453381501619333, + 0.000149758523662058, + 0.00015580764393519075, + 0.00013589719446827075, + 0.0001345001891195352, + 0.00014712084725942987, + 0.00014063341598102852, + 0.00013859630729484707, + 0.00014653197331426782, + 0.00013172351464163512, + 0.0001566219491451193, + 0.00014215871954093018, + 0.000141050991942393, + 0.00015233586367457974, + 0.00014493048752228788, + 0.00011327952176998224, + 0.0001361174226985895, + 0.00013993740117257403, + 0.00014530026760439796, + 0.0001430801164588047, + 0.00012550576184366946, + 0.00014496125118057535, + 0.00015149589944485342, + 0.00014311258252064363, + 0.0001442819162775777, + 0.00013388968488925457, + 0.00015932159431031323, + 0.00015067753213315882, + 0.00014425422159547452, + 0.00016180925081243913, + 0.00015153497088249424, + 0.00013164347654992525, + 0.00015094384406211248 + ] + }, + { + "layer": 18, + "mean_snr": 0.8537423965025113, + "per_head_snr": [ + 0.811851686409326, + 0.8816662652301528, + 0.9021041051052341, + 0.8592093136399362, + 0.904015973097028, + 0.8614903517029873, + 0.8683205610522673, + 0.8365563516836035, + 0.8403269217196677, + 0.8175930308080445, + 0.8584295491939571, + 0.8260228007372978, + 0.8486716447805558, + 0.8355486534147422, + 0.8605445582084661, + 0.8475265772569155 + ], + "mean_band_energy": [ + 2.6612475363717722, + 2.6403484704176954, + 2.7068134025405772, + 2.657537915653651, + 3.3248958402727418, + 3.8540977733186637, + 3.825501650027089, + 4.486317326794722, + 2.6113842987597087, + 2.488405988729009, + 2.777935560836081, + 2.8279631215615435, + 3.3723613708729703, + 4.042692749869542, + 3.91252882565637, + 4.61989204814742, + 5.440452349489474, + 5.663413233775362, + 5.604511678902181, + 5.539473035849857, + 5.712549340747241, + 5.414470627423539, + 5.701549429926564, + 5.485566385358672, + 5.578388998872786, + 5.406197855123272, + 5.668261387686346, + 5.60074703203442, + 5.484108489325095, + 5.450223790148904, + 5.739271644796272, + 5.571075571490721, + 5.700064040935647, + 5.757526921495231, + 5.665582452255347, + 5.719222738556963, + 5.655675105023732, + 5.8249981886079, + 5.5829153917280525, + 5.80269611943973, + 5.551024989575222, + 5.557879838073122, + 5.663607897168173, + 5.381151007558332, + 5.524554288270268, + 5.718797980646368, + 5.600591563556225, + 5.7168747717922646, + 5.590374451082074, + 5.765670427816858, + 5.661299738953625, + 5.6957946410593685, + 5.665367114965832, + 5.4367535212587335, + 5.5327006748516006, + 5.622516063859571, + 5.713027785784023, + 5.6678704673133495, + 5.537670986005539, + 5.654597968587655, + 5.606582889236618, + 5.808604230684104, + 5.614128880829863, + 5.656805028913782 + ], + "dft_magnitude": [ + 322.52311289666545, + 33.96383787948281, + 23.473319150015435, + 12.362717892698406, + 1.1456747543921124, + 7.605787887896762, + 10.794018255882785, + 8.914906510387997, + 6.83744664601077, + 5.638092798415767, + 4.738045740337239, + 2.813316413132681, + 0.4771515909583017, + 3.049889195696807, + 2.886421828902382, + 2.473014332703881, + 2.2229407781216435, + 1.826179964234325, + 2.883045594569155, + 1.7505584302470056, + 1.1655096791957902, + 1.793691730391535, + 3.6244493021226303, + 3.2343405403558285, + 3.373883075047507, + 2.4333495911294745, + 1.5534948495228473, + 1.7378773203722984, + 1.0121839729172966, + 0.4669532959587091, + 2.5757372747425644, + 2.830718730054347, + 2.5492527860551775 + ], + "mean_band_dot": [ + 7.899716493398046e-05, + 9.020715151564218e-05, + 5.3916548438337486e-05, + 4.721608274849132e-05, + 6.624106237040905e-05, + 8.159182505096396e-05, + 8.645751006497449e-05, + 0.0001627609267416119, + 8.638971934260553e-05, + 7.648877110000285e-05, + 6.291885998166435e-05, + 5.3026868442884734e-05, + 7.012570070230595e-05, + 9.993225381776938e-05, + 7.260070231041027e-05, + 0.0001646600192088954, + 0.00018198690105464266, + 0.00018557571178234866, + 0.00017574688394006444, + 0.00017483474391610798, + 0.00019908447052330303, + 0.00018176610694808915, + 0.00018172387490267283, + 0.0001828164190555981, + 0.00017310547536908416, + 0.00018080115114571527, + 0.00017498068893928576, + 0.0001853797964486148, + 0.00017562871414611436, + 0.0001778345603042908, + 0.00018797844018081378, + 0.00017452803768946978, + 0.00018221313061417277, + 0.00019017382021502272, + 0.00018131884417016408, + 0.00017988543834235315, + 0.0001857784777712368, + 0.0001884759130916791, + 0.00018006904974754434, + 0.00018076826130197787, + 0.00019330953296048392, + 0.0001782987675937875, + 0.00018317442879833834, + 0.00016874100481345522, + 0.00016567750162721495, + 0.0001905008649600859, + 0.00017226022004024344, + 0.00018296016719432373, + 0.00019678613307405612, + 0.00019454104170790743, + 0.0001964413993391645, + 0.0001654620235171933, + 0.00020305196983372298, + 0.00016691610547070468, + 0.00018102610493997418, + 0.00019650049875963305, + 0.00019198141967535776, + 0.00019609890065908075, + 0.00018191168828707305, + 0.0001916461137625447, + 0.0001721634694149543, + 0.00018465930497768568, + 0.00018893454364388163, + 0.00018665536435946706 + ] + }, + { + "layer": 19, + "mean_snr": 0.8688942155023641, + "per_head_snr": [ + 0.8768710313942664, + 0.9391036981844183, + 0.8771609397146161, + 0.8062148562110464, + 0.8735969754227402, + 0.8516679565279938, + 0.8301703359596192, + 0.8565275281918595, + 0.8340094085829093, + 0.9333736510369742, + 0.9008255967060849, + 0.8020721958276491, + 0.8645997124891868, + 0.9335213319140352, + 0.8888729912827807, + 0.8337192385916453 + ], + "mean_band_energy": [ + 2.4442763839874964, + 2.5369571531587667, + 2.5536536286690454, + 2.634977435822674, + 3.321669214223621, + 4.112102803233641, + 3.731797141860728, + 4.616430817502203, + 2.4271601174176705, + 2.4694282662684124, + 2.498618138034164, + 2.7283985449431603, + 3.378428068292436, + 4.005374202568222, + 3.8353924704738676, + 4.653585547555467, + 5.507007523909435, + 5.3084934880661, + 5.345844044893302, + 5.319910849602966, + 5.459625168598643, + 5.422802517513165, + 5.4079775890611215, + 5.370400947869822, + 5.4701227921267535, + 5.293406402868021, + 5.413270737824855, + 5.214037437991282, + 5.546065186130049, + 5.377011239707981, + 5.410543226027698, + 5.55340625645487, + 5.311379928372151, + 5.6239564993525075, + 5.579616606786212, + 5.403981120271375, + 5.475086662650126, + 5.460692204758407, + 5.407569188425818, + 5.398285297543455, + 5.4552126628263125, + 5.460199858874361, + 5.44277012552139, + 5.374835550745379, + 5.439974461047086, + 5.5078908994646, + 5.37514510174738, + 5.493656389931626, + 5.4796123041600096, + 5.345329987831301, + 5.271876014172089, + 5.477372155049028, + 5.334737602245576, + 5.448958893697361, + 5.422503699546548, + 5.439074452009866, + 5.442393860847675, + 5.37784231049223, + 5.383456786946512, + 5.318159354588429, + 5.444826753299989, + 5.550167004726002, + 5.4307257706044165, + 5.326093031958311 + ], + "dft_magnitude": [ + 312.0715578831512, + 31.967876965097865, + 21.916686582011568, + 11.26766422513163, + 0.4800698569041885, + 7.880985208363667, + 9.91645659587368, + 10.145897172951333, + 8.17391770339401, + 5.984246275299776, + 3.954059362662712, + 2.7518492895482196, + 0.1744957031703273, + 2.2988169550392494, + 3.442740602413475, + 3.0396962271615253, + 1.171752389751701, + 1.782236487438316, + 2.542711062789184, + 0.9926673078540825, + 0.2564702426058663, + 1.768244248420266, + 4.145862838445704, + 3.166783213220186, + 3.9226482184004, + 2.0714000551009533, + 2.1727402178574935, + 1.1572912770338175, + 0.6408841442940278, + 1.3825011247699206, + 2.7595922957693793, + 3.3205006610732535, + 3.1748799616908343 + ], + "mean_band_dot": [ + 2.182002578976494e-05, + 4.5553897166428214e-05, + 3.2963137925889896e-05, + 2.8809185209865973e-05, + 4.7890250925775035e-05, + 8.627012789474975e-05, + 2.6853257224956906e-05, + 0.00014488943767787532, + 4.496566322131912e-05, + 3.911989624327816e-05, + 2.8677920454356354e-05, + 3.247084038093817e-05, + 4.555069406819712e-05, + 8.280772556190641e-05, + 2.818783335101216e-05, + 0.00016308904764628096, + 0.00016288726988022972, + 0.00013674600404556259, + 0.00015292304942704504, + 0.00015858115921218996, + 0.0001524046622307651, + 0.0001490253843599021, + 0.00016961388257641374, + 0.00015626066033291863, + 0.00015709235708527558, + 0.00013166581334189686, + 0.00014733724765392253, + 0.00013865497041365415, + 0.0001515053088496643, + 0.00015679458016393255, + 0.00015817018493180512, + 0.00015728290850347548, + 0.00013814020257996162, + 0.00015472501866042876, + 0.00014642594294400624, + 0.0001502566221915913, + 0.0001586722507909144, + 0.00015216031027875943, + 0.00014933306283637648, + 0.00015003479484221316, + 0.0001434268651792081, + 0.00015834827968319587, + 0.00015451978765668173, + 0.0001571054746136724, + 0.00014477634567811037, + 0.00015089326603856534, + 0.00016158853281922347, + 0.00016789961250651686, + 0.00015511534866163856, + 0.00014563608806383854, + 0.00014772901704418473, + 0.00014920386684025289, + 0.00013235932192401378, + 0.00014136016655186268, + 0.00014517624356358283, + 0.000140610453172485, + 0.00015002951693077193, + 0.0001535822898404149, + 0.0001473172399073519, + 0.0001461945309984003, + 0.00016010184117476456, + 0.00014314001387560893, + 0.0001419779059403936, + 0.00013686604540907865 + ] + }, + { + "layer": 20, + "mean_snr": 0.8502373194904704, + "per_head_snr": [ + 0.8260872136554851, + 0.7759055442424234, + 0.8415038953966928, + 0.8621312617376542, + 0.8874395176705996, + 0.8910712056316267, + 0.8287195780638483, + 0.8186556593118107, + 0.9031073282313132, + 0.8768606822348505, + 0.8722458906350432, + 0.8232918363648771, + 0.8175057273324088, + 0.876335949631819, + 0.8605280511735681, + 0.8424077705335068 + ], + "mean_band_energy": [ + 3.638581069565012, + 3.6963067185404705, + 3.9211974869899358, + 3.9580283803213705, + 4.624173796491835, + 5.747401915195628, + 5.810646556043174, + 7.031022097892066, + 3.745991598921843, + 3.9303174434309174, + 4.050011348181758, + 4.149697462673133, + 4.5945942313887915, + 5.805546121338967, + 6.227816052020829, + 7.14863613675049, + 9.064305068790397, + 8.85072946679443, + 8.72042760578545, + 8.929168399956295, + 8.808918903423205, + 8.96175487286716, + 8.828700917787298, + 8.627478887521127, + 8.792486944867552, + 9.016991756200351, + 8.893654195226883, + 8.886491813386513, + 8.84766619527328, + 8.64698142169727, + 8.827923638153514, + 8.891663075504997, + 9.139656371447611, + 8.961106323328064, + 8.849635272677181, + 8.8198097067791, + 8.899391099185888, + 8.689679796671502, + 9.192288240546365, + 9.140989595437981, + 8.900791655422838, + 8.888766031892128, + 8.924467126673681, + 8.785473004810942, + 8.800839286990652, + 8.569598290217172, + 8.960979423292098, + 8.77047157328597, + 8.689187707575215, + 9.116834651603995, + 8.968944222885497, + 8.823972335015261, + 9.095651722171556, + 8.790397068479084, + 8.90004508675943, + 8.721935502780585, + 9.047090705539665, + 8.671025461513214, + 9.066355369032776, + 8.651535905847897, + 8.626441099462273, + 9.146553741678098, + 8.538909785067466, + 8.910553652316672 + ], + "dft_magnitude": [ + 503.7346883953698, + 58.26986839581362, + 40.857082960895006, + 20.389220449870503, + 0.40306382227661375, + 12.212388256150515, + 19.024929400030814, + 16.3252087492342, + 10.619097978391514, + 7.986953596377162, + 6.170699888145696, + 4.721313463929277, + 1.0568899606884707, + 3.928068353982607, + 6.1938865184784815, + 5.227047355499054, + 4.351118912070796, + 4.1355651650882965, + 4.775087379819634, + 3.118808936264642, + 0.7843842037821565, + 3.197787200632263, + 4.585781386219292, + 6.468597263662161, + 4.41202864631123, + 3.7184628569056994, + 4.8053434799410155, + 3.7265307995528953, + 0.784746886229662, + 3.4634811386149504, + 4.375832292585427, + 5.649691562319375, + 3.7391488280879344 + ], + "mean_band_dot": [ + 8.114093974143088e-05, + 5.056806271852565e-05, + 2.8185157134430483e-05, + 3.9041626394009654e-05, + 5.349191691550459e-05, + 0.0001222049816078652, + 0.00011584023616251216, + 0.00026968624558776355, + 7.335170820965686e-05, + 8.469617523587658e-05, + 3.122239010622252e-05, + 1.306440663029207e-05, + 3.4561427241897036e-05, + 9.131097283443523e-05, + 0.0001452189266615278, + 0.0002594489437797165, + 0.00023371669067273615, + 0.00023487011412726133, + 0.00022549224877366214, + 0.00023205815477922442, + 0.0002175671243094257, + 0.00023396932374453172, + 0.0002102411696114359, + 0.00021639191027134075, + 0.0002457170653542562, + 0.00023759174609949696, + 0.0002232771699937075, + 0.00022206029984772613, + 0.00020239742252670112, + 0.0002289458302584535, + 0.00026562005314190174, + 0.00020620078385036322, + 0.0002551433130975056, + 0.00024685634571142145, + 0.00023655178983972291, + 0.00024248411557437066, + 0.0002516272375032713, + 0.00022944291049498133, + 0.0002482720651642012, + 0.0002319443467513338, + 0.00022929664783077897, + 0.0002407735137239797, + 0.0002217687203085461, + 0.00023111679456633283, + 0.0002301283915642216, + 0.0002215125527982309, + 0.00023374955515009788, + 0.0002058076702269318, + 0.0002298125880315638, + 0.00022427431929372688, + 0.00023664482387175667, + 0.00025131742540907, + 0.0002451887344818715, + 0.00021169236617879506, + 0.0002346574094076459, + 0.00024655018074781765, + 0.0002244315242023731, + 0.00024652304773553624, + 0.00024020237987087967, + 0.00024044242968557228, + 0.00022030559875929612, + 0.00023681075253989547, + 0.00021961387051305792, + 0.0002459616521264252 + ] + }, + { + "layer": 21, + "mean_snr": 0.848010565484884, + "per_head_snr": [ + 0.802080407515088, + 0.8409576811746666, + 0.8469272901479697, + 0.8620875406060738, + 0.8787986549646803, + 0.8396384845763121, + 0.8664592663240132, + 0.8818456173760196, + 0.8551928690368039, + 0.7947159981641467, + 0.8416030607683953, + 0.8671471902620852, + 0.8425787808337024, + 0.8008240818178033, + 0.8248506553177777, + 0.9224614688726063 + ], + "mean_band_energy": [ + 3.7173843302469853, + 3.8197493592329117, + 3.995306617941117, + 3.8661328215696473, + 5.017643392826517, + 5.931327775167929, + 5.466429705272657, + 6.872735840766879, + 3.835301125200264, + 3.9256703970321123, + 3.95797791129883, + 4.289843805054601, + 5.111072761699746, + 5.655199496774863, + 5.494449092929823, + 6.966298900437341, + 8.46361093600257, + 8.456514366974002, + 8.919523940646668, + 8.655309088680687, + 8.63514802002128, + 8.518043625664827, + 8.729782734527259, + 8.524243587498843, + 8.645200968833388, + 8.71102612934278, + 8.679615376987098, + 9.014374575301574, + 8.792521215564108, + 8.916263055660277, + 8.47420228576696, + 8.095674950778593, + 8.754047326607324, + 8.407135938862746, + 8.369290548617753, + 8.557831260182024, + 8.868696538397037, + 8.408705444092341, + 8.83884636628973, + 8.509306668877137, + 8.406661048537439, + 8.188075836818422, + 8.723229026711955, + 8.896625283684866, + 8.84584645911541, + 8.702466910067937, + 8.533602303492733, + 8.544052018245123, + 8.611878185629724, + 8.665367816375271, + 8.687149382674153, + 8.67048924595566, + 8.640696973171448, + 8.43686160564027, + 8.40622806948629, + 8.69371314585367, + 8.537739983777584, + 9.152699585092853, + 8.842656389883928, + 8.55947523562886, + 8.754068737429883, + 8.881370733814501, + 8.830728158866123, + 8.66280735213683 + ], + "dft_magnitude": [ + 492.74192777172016, + 53.36782904271759, + 40.238549807266665, + 18.829173754678106, + 2.5077637346237704, + 12.836461215781164, + 18.181228150221955, + 14.327143282170164, + 9.29337741753288, + 7.589632378722679, + 9.064601524832048, + 5.384551703522632, + 0.7076390976797159, + 5.218938755802826, + 6.0129330708550475, + 4.853821298524192, + 2.9142243570475355, + 3.2771185293620215, + 1.5435929886300688, + 3.604495017828967, + 0.6203348854299233, + 2.6984896885460437, + 5.215311780663319, + 7.836407615865573, + 5.1429228957436495, + 4.750728459633616, + 1.8591732518483042, + 1.0644833556293432, + 1.6368461299103556, + 2.795635536242208, + 2.7109860086065938, + 5.734206971278935, + 3.568855942812604 + ], + "mean_band_dot": [ + 3.324644126223575e-06, + 2.283239410871829e-05, + 4.53918150356003e-05, + 3.389156165667373e-05, + 5.578601565048302e-05, + 0.0001042493434511016, + 7.685938044232898e-05, + 0.00021513123760996677, + 3.4891781879764494e-05, + 4.473609777733145e-05, + 3.7494771120805126e-05, + 2.9082377537292814e-05, + 7.149677992401848e-05, + 0.00010832916279923666, + 7.153373030632793e-05, + 0.00022860740227770293, + 0.00019964053967669315, + 0.000209450048259896, + 0.0001717451613103549, + 0.00019399141115172824, + 0.00018881340827192616, + 0.0001978664513444528, + 0.00021342372906474338, + 0.00021165975499570777, + 0.00019780364755206392, + 0.00017925337897395366, + 0.00018613443535286933, + 0.00018695062772167148, + 0.00019306554986542324, + 0.00015392256432278373, + 0.00019444892927822366, + 0.00018473568843546673, + 0.00023488091437684488, + 0.00018439730160935142, + 0.00020010774642287288, + 0.0001986624578194096, + 0.0001978871277970029, + 0.00017966434552363353, + 0.00018110622090716788, + 0.0001927948474076402, + 0.00018517447222166084, + 0.00022588003821510938, + 0.00019882906599377748, + 0.0002113695902608015, + 0.00020695693683592253, + 0.0002124988560581187, + 0.0002062220358993727, + 0.0001915528523568355, + 0.0002095380032187677, + 0.00021932962317805504, + 0.00018429795534302684, + 0.0001754431868903339, + 0.00021420130633487133, + 0.00018206906759132835, + 0.00021430822926049586, + 0.00020231320240782225, + 0.00020414356833953207, + 0.000205175568339655, + 0.0002069250072054274, + 0.000199979260287364, + 0.0001969712299967341, + 0.00021581548276117246, + 0.00020144406028066442, + 0.00021751835606664827 + ] + }, + { + "layer": 22, + "mean_snr": 0.8417170421027536, + "per_head_snr": [ + 0.840022315035151, + 0.8470674124984655, + 0.8650700385155575, + 0.8775202093271424, + 0.8032050909418651, + 0.8173735956677977, + 0.8347218277016464, + 0.8429046994531393, + 0.8777072988556076, + 0.8633318328220008, + 0.8367686862751491, + 0.8674400068377623, + 0.8487667135484408, + 0.7928081531413232, + 0.8096578598939321, + 0.843106933129076 + ], + "mean_band_energy": [ + 4.293366301304038, + 4.1702146168505605, + 4.361045799570318, + 4.752912986478707, + 5.816533210165421, + 6.848580472330083, + 7.142483144663261, + 7.927006735389888, + 4.221241529970916, + 4.407646247650223, + 4.69577586361988, + 4.73325944787373, + 5.828985240477362, + 6.7597692042691975, + 6.798453397361655, + 8.339618754288487, + 10.611698665082873, + 11.11718987617088, + 11.008528120237635, + 10.564330764428956, + 10.443077511901517, + 10.544511308209234, + 10.859429285753407, + 11.018165026361853, + 11.365981768830007, + 10.751575087600731, + 10.846548331759045, + 11.476843326746657, + 10.668272432843079, + 11.200914110249535, + 10.39553652444474, + 10.4207731587363, + 10.402709122502031, + 11.035330987210301, + 10.821019462258437, + 10.943834171509899, + 11.042380047279, + 10.865183611379152, + 11.100793839368869, + 11.004061743007473, + 11.186396588840335, + 11.077209889606852, + 11.376976354309022, + 11.109720358127607, + 10.585406425303141, + 10.917315714355865, + 9.995294075923587, + 10.107372128389486, + 10.941270034686841, + 11.196871558461474, + 10.797154897326305, + 10.711882745020496, + 10.532343889043759, + 10.67876208417135, + 11.082945432295311, + 11.488248398253535, + 10.566027852911773, + 10.723736129413902, + 10.533650325094314, + 10.822617028441357, + 11.280255439823549, + 11.111663547392673, + 11.037990522233947, + 11.019181721260935 + ], + "dft_magnitude": [ + 612.4858743768228, + 74.5780765798303, + 52.21885175426777, + 28.017679127047973, + 3.8326096010417325, + 12.969713104776968, + 24.36809720033767, + 22.699063072872217, + 11.587514005150393, + 12.979085566933747, + 13.335531682664078, + 5.736969349788933, + 4.398237806718081, + 3.9763117620826387, + 6.514748099257603, + 7.102635478683606, + 4.314150341351363, + 5.463071233888237, + 7.30587980000101, + 3.7032604992213076, + 1.5089731618251454, + 2.252357544214813, + 6.875210535585963, + 8.376225304800508, + 5.248829773589734, + 6.057832606153377, + 3.5385316662796233, + 0.20633166176437723, + 2.5578989795704645, + 2.8116171670362156, + 4.40937608380704, + 4.1946477596120015, + 7.206731502452044 + ], + "mean_band_dot": [ + 3.709754241754126e-05, + 6.470254083978944e-05, + 2.5758787273844064e-05, + 2.4991890199999034e-05, + 6.82809151157926e-05, + 0.0001272116949166957, + 0.00015542495145837165, + 0.0003571904821910721, + 3.2666402944414585e-05, + 1.2488134018440178e-05, + 1.5236406454732787e-05, + -1.3114927497781537e-05, + 5.3091929828497086e-05, + 0.00013557512784245773, + 0.0001484410946659409, + 0.00036733179877046496, + 0.00031514195643467247, + 0.000320422350341687, + 0.00037933565101866407, + 0.00034264659012706034, + 0.00030556772071577143, + 0.00033464012778949837, + 0.0003694968909258023, + 0.0003332511460598653, + 0.0003361298611253005, + 0.00028265515970815613, + 0.00032664821696926083, + 0.00034658732738535036, + 0.00035645519147919913, + 0.0003371410124373142, + 0.00030559529977836064, + 0.00029320645091956976, + 0.0003402398729122069, + 0.0003159070731726388, + 0.00033320364582323236, + 0.00030710053192706255, + 0.000336996356054442, + 0.0003463639957317355, + 0.00032666818879079074, + 0.00033683804167594644, + 0.00035353798591586383, + 0.0003411829786728049, + 0.0003220898333893274, + 0.00032324868345767754, + 0.0003278258951979751, + 0.00032814586938911816, + 0.00029863436191135406, + 0.00032767956446377866, + 0.00033796635938188047, + 0.00032596094388281927, + 0.00032785813311875245, + 0.00031261408196314733, + 0.0003134207253197019, + 0.0003080979297465092, + 0.00030975627976204123, + 0.0003327271545003896, + 0.00032794916342027136, + 0.00033036195668501023, + 0.00034230418805236695, + 0.000329955313929986, + 0.0003286086916318709, + 0.0003359046806963306, + 0.00028665951151651825, + 0.000316859439038808 + ] + }, + { + "layer": 23, + "mean_snr": 0.8469096271208884, + "per_head_snr": [ + 0.8722857765898266, + 0.7970899827628637, + 0.8768453316889895, + 0.8789481829655075, + 0.845430068496627, + 0.9256527320387883, + 0.8411449425351303, + 0.7931220173637403, + 0.8256095866323119, + 0.8479519010669367, + 0.8785553952836008, + 0.8422985441284924, + 0.7914475664190757, + 0.8530255435405348, + 0.8410734759659796, + 0.8400729864558083 + ], + "mean_band_energy": [ + 4.492238989743011, + 4.275483099658607, + 4.689068604248277, + 4.995550943922568, + 5.972252650064156, + 6.691197621322603, + 6.382651054882864, + 7.725067075690038, + 4.540106858259818, + 4.377592742025923, + 4.209436861305723, + 4.339379865560155, + 5.789619232095555, + 6.6367686623182145, + 6.046999208678637, + 7.750219988822952, + 10.109353794418087, + 9.818770472195375, + 9.721205482039792, + 10.149047711214802, + 9.916173102585613, + 10.10734700901775, + 10.26580700193545, + 10.088341415614142, + 9.655396180653067, + 9.64249856845349, + 10.993058983352704, + 9.986094417848484, + 9.879288258866115, + 10.049469133696302, + 9.91415335270489, + 9.496614544946201, + 9.72396387220466, + 9.544964759079402, + 9.379170083866743, + 10.29058644038549, + 10.122640931095262, + 9.808005474862373, + 9.65385656313605, + 9.559951679835258, + 9.799695130334968, + 10.435227826128145, + 9.862914194595714, + 9.587433843897443, + 9.518974063416774, + 9.576223084307319, + 9.235603580132732, + 10.161026338549974, + 10.205065341481966, + 10.09945148782666, + 10.132991377236781, + 9.981039631214262, + 9.973916940377379, + 9.723525448159922, + 9.833703480422987, + 9.842961808766635, + 9.886831320277462, + 9.581728818832108, + 10.007800544808003, + 9.926149056398703, + 9.971971988851998, + 10.028850126527757, + 10.291630046258849, + 8.989837664315221 + ], + "dft_magnitude": [ + 563.4439458357263, + 61.55763828976029, + 47.16501746269947, + 20.417293301399546, + 2.41009816370826, + 12.660410911241181, + 22.84574137098477, + 16.481396064502388, + 9.64128113557767, + 10.829017060626882, + 7.9735308615447895, + 2.9703332033334977, + 1.3205895241704524, + 7.6515858432254085, + 8.619310627703861, + 4.612096374231343, + 2.6908736338026236, + 2.900264892030779, + 5.8109970996589615, + 4.337404965111802, + 3.9107106967657956, + 5.937983318688235, + 4.606402610282455, + 6.32541163581468, + 4.555692107397887, + 2.451530349138145, + 2.4347694698119606, + 2.808659482608573, + 1.2358141829723026, + 1.644793790225714, + 4.622286862927448, + 2.8211695654339, + 3.0888676870621907 + ], + "mean_band_dot": [ + -0.00010246583860862302, + -5.658444649725425e-05, + -1.9678260400723957e-05, + -2.352921399051411e-06, + 6.134683513892014e-05, + 0.00011032738825633714, + 3.9808568857324644e-05, + 0.00028972603286092635, + -9.518017749599039e-05, + -1.5519752025738853e-05, + 5.007966024095367e-06, + 1.1055931281589437e-06, + 5.35721704864045e-05, + 0.00010798199656392171, + 6.792604187921825e-05, + 0.0002869335849027266, + 0.0002547536515749016, + 0.00023463564025405503, + 0.00023135108199312526, + 0.00018426658925818629, + 0.00021915828551755112, + 0.00020821089628952905, + 0.00019466400391365823, + 0.00018790430340231978, + 0.00019994925946775766, + 0.000253681491358293, + 0.00021713499290854088, + 0.00020150948989794415, + 0.00020830757216572238, + 0.00022610039104620228, + 0.0002317519008556701, + 0.00022405267031899712, + 0.00024267172057079733, + 0.0002103734481124775, + 0.0001951076486079728, + 0.00021692502377845813, + 0.000252556138093496, + 0.0002367393608437851, + 0.00023714171075539525, + 0.00019542734278843454, + 0.00023156297288551286, + 0.0002098232997695959, + 0.00023889059912107768, + 0.00021837266717739112, + 0.0002224750023742672, + 0.0002283296571476967, + 0.000249174267764829, + 0.0002682303875189973, + 0.000229840494966993, + 0.00026184084663327667, + 0.00020303397047882754, + 0.00026553517932370596, + 0.00023175401702246745, + 0.00025092582473007496, + 0.00022590562957702787, + 0.00022334268578561023, + 0.00022004251559337717, + 0.0002289492385898484, + 0.0002481627584529633, + 0.00022293169354270503, + 0.0002100155717243979, + 0.00023646048703085398, + 0.0002056614957837155, + 0.00021470152395863806 + ] + } + ], + "elapsed_s": 1.2807104587554932 +} \ No newline at end of file diff --git a/data/exp_wqk_spectral/pythia-1b_wqk_spectral.json b/data/exp_wqk_spectral/pythia-1b_wqk_spectral.json new file mode 100644 index 0000000000000000000000000000000000000000..5c3de16ec611ebcae6deb2004508f3b2c112d4fb --- /dev/null +++ b/data/exp_wqk_spectral/pythia-1b_wqk_spectral.json @@ -0,0 +1,5471 @@ +{ + "model": "EleutherAI/pythia-1b", + "short_name": "pythia-1b", + "theta": 10000, + "T_train": 2048, + "d_head": 256, + "n_pairs": 128, + "k_dead": 91, + "global_mean_snr": 0.9003423335245075, + "global_min_snr": 0.8769781560117472, + "global_max_snr": 0.9311415592176637, + "layers": [ + { + "layer": 0, + "mean_snr": 0.8834016080175577, + "per_head_snr": [ + 1.065950858559742, + 0.8620830878281959, + 0.8794173816887588, + 0.820560946223679, + 0.8848578786708032, + 0.8434422187046277, + 0.8862385748640135, + 0.8246619176006423 + ], + "mean_band_energy": [ + 0.22821045034477833, + 0.25771407677525715, + 0.27025926079033596, + 0.3842593679447939, + 0.3899902820001987, + 0.41177011319762324, + 0.44244067506726514, + 0.49952116814790215, + 0.5763672221191343, + 0.6485147412716481, + 0.7698296318782972, + 0.8687411792568809, + 0.7464036559848468, + 0.8860730603522442, + 0.9668938048139202, + 0.9672864597165826, + 0.21818302558908775, + 0.25313657618254537, + 0.2778431533185154, + 0.3589494647915216, + 0.39256006520038467, + 0.4417557545118961, + 0.44733565291314503, + 0.4187491815487815, + 0.6231456802449129, + 0.5969023568544869, + 0.7882394461632295, + 0.7503692004868296, + 0.8158386735985244, + 0.9148445728295713, + 0.9076193505955973, + 0.9607015704116906, + 0.9788519430890792, + 0.9955331286730971, + 0.9932023550799203, + 1.001202263201792, + 0.9771259143426168, + 0.9675118170412897, + 0.9864572399829395, + 0.970244007019153, + 0.9939079711132243, + 0.9697293151717106, + 0.9897491018378912, + 0.9721224272236002, + 0.9825913730412372, + 0.9593764912178977, + 0.9907241066611704, + 0.9774317507716443, + 0.9741748003357432, + 0.9038809985798075, + 0.959907123519274, + 0.9862528684735152, + 0.9867892592082876, + 0.8914092607443891, + 0.9911257173497927, + 0.9711581506829972, + 0.9765800802733702, + 0.9924610039473195, + 0.9773426286038842, + 0.9781662738833103, + 0.955078680184061, + 0.9774993514693424, + 0.9662538829523878, + 0.9633723056655438, + 0.9769162062446575, + 0.9680866240759118, + 0.9840246429165815, + 0.9600787766820051, + 0.9667655583018575, + 0.990285929342514, + 0.9688172021165937, + 0.989954294568411, + 0.971154014032279, + 0.9754599305863612, + 0.9676801840598026, + 0.9798312794397512, + 0.9842872481499447, + 0.9777934982040559, + 0.9733594604305122, + 0.9627647550769098, + 0.9622973294093833, + 0.9696298044941707, + 0.9667544508419552, + 0.983237234727198, + 0.9671859729759313, + 0.9706981580247259, + 0.9820330549628947, + 0.9482795236602413, + 0.957834079842179, + 0.9800471299030313, + 0.9853029497173837, + 0.9651426196823112, + 0.9944296033042376, + 0.998290818407793, + 0.9922435811745501, + 0.9668752720974827, + 0.9851909344690406, + 0.9690363948069828, + 0.9541848742166056, + 0.9942935119831122, + 0.975894403559505, + 0.9658656772387426, + 0.9978269020504579, + 0.9733731734055977, + 1.0116801572401601, + 0.9927356474808672, + 0.9832064772488447, + 0.9770521160381311, + 0.9836097625684459, + 0.9911685402892503, + 0.9935818183964731, + 0.9846618140359737, + 0.9775235736208909, + 0.9793687870896086, + 0.963006071538848, + 0.9883939623473251, + 1.0098722146010943, + 0.9793449173509305, + 0.9958878041497518, + 0.9689600669821918, + 0.9751656547531331, + 0.9315555922812591, + 0.9773280678382572, + 0.9893502044164462, + 0.9779892047241289, + 0.982374569847889, + 0.979885155204088, + 0.9803286440266075 + ], + "dft_magnitude": [ + 112.21490034949798, + 11.44428726590489, + 8.585905355516616, + 4.29607884629914, + 0.32412446704810133, + 3.506148576552196, + 5.164053896965395, + 5.2377003341706025, + 4.684182349312737, + 3.2112248838683546, + 1.9299702518786368, + 1.2814916085553167, + 0.1315242488193556, + 0.9444737536845387, + 1.428689104689835, + 1.9321713929783815, + 2.0196876382363538, + 1.719040524657643, + 1.317247484351053, + 0.5094400762990478, + 0.24099493763118277, + 0.9565390227333755, + 1.223219260391083, + 1.599760679047488, + 1.8190713458689698, + 1.408004617778529, + 0.7767169611200843, + 0.602368798066604, + 0.2415482692938189, + 0.6417001938165315, + 0.6515595293115736, + 1.1399914709171264, + 1.3073022208193141, + 1.0922675467037768, + 1.1456840304361935, + 0.7883774244130015, + 0.24236485757696802, + 0.1215011839233245, + 0.7321035599964724, + 0.8688030216397017, + 0.6393047498464224, + 0.8623511429697978, + 0.7527964260583575, + 0.31973307555362696, + 0.18639701940780645, + 0.26320876445338554, + 0.7022151430312121, + 0.6735431344038415, + 0.3534520054218974, + 0.3044946139587785, + 0.31045842568995996, + 0.460066367351293, + 0.16123230723561427, + 0.2663132001667358, + 0.2575201634169367, + 1.0884569709477985, + 1.1943296722970385, + 0.9459939824765308, + 0.6702773320846267, + 0.22923859952288614, + 0.3999001865609328, + 0.36860813585028296, + 0.9289650142459497, + 0.8034548281331635, + 0.4470187037829305 + ], + "mean_band_dot": [ + -1.0403633439182158e-05, + -1.4898137024488278e-05, + -5.605846212120014e-06, + -1.1502665152818281e-05, + -1.0315161887319846e-05, + -2.7430976103914873e-06, + -1.0433964021672182e-05, + -1.0921103267946819e-05, + -1.4402716089989553e-05, + -1.7482450857642107e-05, + -2.5910252048788607e-05, + -1.586873395353905e-06, + -1.1429668433038387e-05, + -2.1858442380562337e-05, + -2.5466544428809357e-05, + -2.3147521396538195e-05, + -9.459016045099133e-06, + -1.15015175410349e-05, + -8.409194293790279e-06, + -1.0253249104863471e-05, + -7.1311649207927985e-06, + -8.178432096883625e-06, + -4.537994463760242e-06, + -8.901491909796277e-06, + -1.3428539432425168e-05, + -1.2045332368870731e-05, + -2.5554987246323435e-05, + -4.645198970365527e-06, + -1.496940049605655e-05, + -2.1928618963329427e-05, + -1.891097156203614e-05, + -2.5408176753671796e-05, + -2.2571472641175205e-05, + -2.7622858311815435e-05, + -2.857883512774606e-05, + -2.92512933128819e-05, + -2.8220817569035717e-05, + -1.977473962710974e-05, + -2.4119529257404793e-05, + -2.5205648679360593e-05, + -2.6785908403326175e-05, + -2.776249169755829e-05, + -2.956192793135415e-05, + -2.2147724017429482e-05, + -2.6134369477404107e-05, + -2.298306816328477e-05, + -2.510360400265199e-05, + -2.549900733583854e-05, + -2.371664083966607e-05, + -2.0180222918497748e-05, + -2.3661034617816767e-05, + -2.744064704529592e-05, + -2.6416736830014997e-05, + -1.6927704280078615e-05, + -2.390206435620712e-05, + -2.599710438744296e-05, + -2.502533357073844e-05, + -2.7205157664411672e-05, + -2.4784251024811965e-05, + -2.242310538669301e-05, + -2.633727254419682e-05, + -2.5545664357196074e-05, + -2.2449705681992782e-05, + -2.809041933460321e-05, + -2.667662454314268e-05, + -2.3359595246574827e-05, + -2.4511591846021474e-05, + -2.3417066984166013e-05, + -2.370120930095254e-05, + -2.397825818434285e-05, + -2.5481533498350473e-05, + -2.9010911148930063e-05, + -2.2398095651965377e-05, + -2.448538407406886e-05, + -2.364629153817077e-05, + -2.8698891256340175e-05, + -2.1546888433476852e-05, + -2.7643809460187185e-05, + -2.268994967380422e-05, + -2.060155173921885e-05, + -2.2785918417866924e-05, + -2.284487980830363e-05, + -1.908994227051153e-05, + -2.6154689010127186e-05, + -2.6490918259014506e-05, + -2.2765572509797494e-05, + -2.767129757330622e-05, + -2.2290930075996584e-05, + -1.7250495147891343e-05, + -2.722128428445103e-05, + -2.8368948491674928e-05, + -2.6005427443465123e-05, + -2.8073187735344618e-05, + -2.483618465021209e-05, + -2.3985387144875858e-05, + -2.4710090769985982e-05, + -2.5973244419219554e-05, + -2.4915482413234713e-05, + -2.5329579614208342e-05, + -2.4920489977375837e-05, + -2.7258008600483663e-05, + -2.3746942048319397e-05, + -2.6045537595109636e-05, + -2.090647492991593e-05, + -2.740329887274129e-05, + -2.2315865976452187e-05, + -2.5681295255708392e-05, + -2.1547291204626617e-05, + -2.8099379278501146e-05, + -2.5711572561704088e-05, + -2.674205870789592e-05, + -2.5715065078202315e-05, + -2.7367307097847515e-05, + -2.859094291807196e-05, + -2.5567788348723752e-05, + -2.5278024086361484e-05, + -2.3208658490148082e-05, + -2.4647375198583177e-05, + -2.7723808386781457e-05, + -2.291579659186027e-05, + -2.3157572712761976e-05, + -1.8269788846225765e-05, + -2.462331619312863e-05, + -2.8132114842094325e-05, + -2.9257879617716753e-05, + -2.7045327158248256e-05, + -2.705577651340718e-05, + -2.562136849348917e-05 + ] + }, + { + "layer": 1, + "mean_snr": 0.9311415592176637, + "per_head_snr": [ + 0.8939440820915494, + 1.0434603218617904, + 0.861804128446189, + 0.9095297460354314, + 0.8320608917478063, + 0.9119435727306865, + 1.0859102326379872, + 0.9104794981898697 + ], + "mean_band_energy": [ + 0.7935397949503226, + 0.7278042916023237, + 0.9166038376971031, + 0.9346621484078388, + 0.8178957406629362, + 0.8643725395057538, + 0.9537829680985423, + 1.1988275445329868, + 1.0479011298391874, + 1.2607997530646955, + 1.69957457402964, + 1.436288940068681, + 1.4860417499213812, + 1.7384539538881967, + 1.7930121219654307, + 1.771564894775194, + 0.7242093898436242, + 0.729954108392247, + 0.935707261094513, + 0.8389996875108747, + 0.7855188417086141, + 0.8751909505014841, + 0.9460559946645475, + 1.283219895908093, + 1.0816625659938186, + 1.4530294063371318, + 1.6126668292862432, + 1.5731887616202682, + 1.5492605288364292, + 1.7712970832983692, + 1.7802264089476711, + 1.8100863015672588, + 1.7538714676651876, + 1.739177794169808, + 1.794850096434053, + 1.7925060619599895, + 1.8085992832487512, + 1.794277549947907, + 1.8078771170304555, + 1.8059198609390203, + 1.82040900911448, + 1.8025565028381512, + 1.8196838666331994, + 1.8039100083599502, + 1.7911644424760844, + 1.8046507805964058, + 1.8070109742965066, + 1.8075182447056628, + 1.8321869598934688, + 1.8089643378199232, + 1.8038711304598407, + 1.816959728830847, + 1.7757384014318056, + 1.7487648354039158, + 1.8527804448657519, + 1.8183293761429349, + 1.8015636216398097, + 1.8182258490519119, + 1.7825165058972745, + 1.7986203887937657, + 1.7999692106815375, + 1.819927185971121, + 1.806572313900726, + 1.815201301823254, + 1.812273382590945, + 1.7888826902995723, + 1.7944985402709457, + 1.8376711290406562, + 1.825397970553765, + 1.8117002023354805, + 1.8067551412709095, + 1.8303985315606277, + 1.8051787088616509, + 1.780166270490719, + 1.813219615058288, + 1.7840796243691175, + 1.8047913385370626, + 1.8115184727032685, + 1.8133152068901914, + 1.801611933308111, + 1.8145278726666163, + 1.8145263634120843, + 1.7668167244395292, + 1.7946999533502122, + 1.8239727478616117, + 1.7954397628891412, + 1.837298461537408, + 1.7948682172853863, + 1.8131289610133283, + 1.7954844904485032, + 1.8137922191844034, + 1.8460402230881807, + 1.81491065057509, + 1.8038530995845568, + 1.8290229938377545, + 1.836286149808867, + 1.822268990996101, + 1.8314819338317294, + 1.8154310726599072, + 1.755002659938762, + 1.8087159622215299, + 1.8034175756388073, + 1.8072327466794942, + 1.7982838401432772, + 1.7988350671148101, + 1.797970416131197, + 1.7995473632763321, + 1.8227449181408808, + 1.7741092546589585, + 1.8345958074957123, + 1.8338386531794173, + 1.8189380040932233, + 1.8169425531215633, + 1.8005156935220672, + 1.8397137415149607, + 1.8057852640667234, + 1.8016400095807992, + 1.8453294359579004, + 1.8143341309332999, + 1.7616153516527486, + 1.8034306484058358, + 1.8075621304731655, + 1.7989664433246246, + 1.7875220175904118, + 1.799624796513354, + 1.749137940718008, + 1.8053502323610813, + 1.7834717634436563 + ], + "dft_magnitude": [ + 212.50505872004922, + 17.24107996510545, + 12.639979804307817, + 6.405177468598022, + 0.3665660203040775, + 4.8228494859868665, + 7.302183697855416, + 8.013891442113202, + 7.3518044227068975, + 5.558403797395165, + 3.1791527052512727, + 1.3749036375612909, + 0.34571429421540717, + 1.1421075583179645, + 1.8902521115133344, + 2.1429094389711945, + 2.3371044555683156, + 1.6907434943930215, + 1.3926378690462986, + 0.43728533479265635, + 0.15190318974669442, + 0.45951803766462124, + 1.7300574282803345, + 2.04384659964679, + 2.271335506321852, + 2.1716448318761277, + 1.2158608899885013, + 0.5508600219284815, + 0.21411594550027604, + 0.6597817681738106, + 1.4265294583905463, + 2.496865741525268, + 2.9277874282361123, + 2.536783281439302, + 1.9956625118015674, + 1.347175130252357, + 0.46310448066309406, + 0.41305171470981006, + 0.620572609444809, + 0.9804085449875344, + 0.308399076932202, + 0.6435460268288284, + 0.665207222620603, + 0.42918022623347835, + 0.4768489440910288, + 0.8078217097565216, + 1.3918312213749804, + 1.7442589008616554, + 1.6216263081010571, + 1.5883430036298847, + 1.1223281473503675, + 0.40044264936701757, + 0.41091315499895675, + 0.6879144091570869, + 0.6331895616444541, + 0.4454242560388691, + 0.6001344779827451, + 0.5007500096812052, + 0.7457498369198141, + 0.5631609421800007, + 0.35683126820412925, + 0.4218766786684044, + 0.7284993892836251, + 1.1422195067980063, + 1.0826451502482257 + ], + "mean_band_dot": [ + -4.4258365505811526e-06, + -9.013196518026234e-07, + 2.0941351863257296e-05, + 5.819261403416931e-06, + -2.655736846435275e-06, + -1.7872695252663107e-05, + -2.0104957911826205e-06, + 1.204177425506714e-05, + -1.7011450665904704e-06, + -1.0996061604373608e-05, + -7.502598054998089e-05, + -1.7567147111208214e-05, + -3.361684132130449e-05, + -4.9843147962747025e-05, + -3.476476510400062e-05, + -3.6486774092736596e-05, + -1.5749840201806364e-05, + 2.1859672045820844e-06, + 3.280567011643143e-05, + -1.3497591453415225e-06, + -1.3453518022288335e-05, + 3.2368521374337433e-07, + -1.4279239294978652e-05, + -1.452338062790659e-05, + -4.912363252174146e-06, + -1.5852257313042628e-05, + -2.2099242485751347e-05, + -1.6841500283248934e-05, + -4.1754958658657415e-05, + -3.518567007176898e-05, + -2.4890391642884424e-05, + -3.480999441762833e-05, + -3.9351991517833085e-05, + -4.349108655787859e-05, + -3.4163530713726686e-05, + -2.9853344983621355e-05, + -3.156388281055911e-05, + -3.669700265618303e-05, + -2.7316889656958665e-05, + -3.1459317028748046e-05, + -3.613883541220275e-05, + -3.4036821944027906e-05, + -2.3498771980712263e-05, + -3.444203258595735e-05, + -2.7690995196394397e-05, + -2.6224880116387794e-05, + -3.018780415686706e-05, + -3.2826524773099663e-05, + -3.626735741590892e-05, + -3.7392541287317727e-05, + -2.8913581786582654e-05, + -3.200026679905932e-05, + -2.7470502086401893e-05, + -4.134872415306745e-05, + -3.0821498441468975e-05, + -2.9125336823199177e-05, + -3.0948318539003594e-05, + -3.764856876387057e-05, + -3.395182255871987e-05, + -3.0352545820733212e-05, + -3.770044976292297e-05, + -3.2090359027847626e-05, + -3.2398672658473515e-05, + -3.1008100762619506e-05, + -3.461466315002326e-05, + -3.18105396104329e-05, + -3.115411571741333e-05, + -3.9799384524030756e-05, + -3.623075565428735e-05, + -3.0489730065141885e-05, + -3.756561450529716e-05, + -3.3759244303155356e-05, + -3.09037170609372e-05, + -2.756795629466069e-05, + -3.734984085213e-05, + -3.4239335036545526e-05, + -3.4018832280935385e-05, + -2.7653628805524022e-05, + -3.175121103993206e-05, + -2.9970966096470875e-05, + -3.823586141038504e-05, + -3.544470308725067e-05, + -4.368131007481679e-05, + -2.978596441494119e-05, + -2.8091967919863237e-05, + -2.8077180729724205e-05, + -2.897392244705088e-05, + -3.624724346451558e-05, + -3.385581936754534e-05, + -3.540083156394758e-05, + -3.380227667548752e-05, + -2.9501381845875585e-05, + -3.4452299331633185e-05, + -3.691727698651448e-05, + -2.9590564963655197e-05, + -2.6387694902041403e-05, + -2.89595869560344e-05, + -2.883587785618147e-05, + -3.431671865428143e-05, + -4.2151332991124946e-05, + -3.1805919689986695e-05, + -3.7195900802089454e-05, + -2.851606089393499e-05, + -3.1363357237523815e-05, + -3.8335281004719945e-05, + -2.705018222570743e-05, + -3.37286295746253e-05, + -3.120821216384684e-05, + -3.958020572270016e-05, + -2.9276952062673445e-05, + -3.745606363736442e-05, + -3.389488870197965e-05, + -2.7821543273631733e-05, + -3.063691496407728e-05, + -2.7121098923998943e-05, + -3.884614974936085e-05, + -3.228651800668558e-05, + -3.2530393013985304e-05, + -3.207676621741484e-05, + -4.6800957363757334e-05, + -3.1025706391574204e-05, + -3.374683764434394e-05, + -2.7672106597265156e-05, + -2.626784007020433e-05, + -3.086144522512768e-05, + -4.1798481163368706e-05, + -3.0312374917684792e-05, + -2.7952807784004108e-05 + ] + }, + { + "layer": 2, + "mean_snr": 0.8966648525794749, + "per_head_snr": [ + 0.8941824420994173, + 0.893330113216318, + 0.906868321886776, + 0.8741561395965433, + 0.8893596439416575, + 0.9100973038148595, + 0.9009404246613113, + 0.9043844314189167 + ], + "mean_band_energy": [ + 1.0186327669726207, + 0.8801604551220681, + 1.1197173026379805, + 1.1608068982026278, + 1.0265382022189762, + 1.136826897896266, + 1.3015396319629193, + 1.3861010414152979, + 1.4812112739192527, + 1.3918800266023004, + 1.814216671642697, + 1.3840356417985378, + 1.5819593161433811, + 1.9820365749094524, + 2.013056408494152, + 1.9895183114100483, + 1.1076873300191208, + 0.9418893354228701, + 1.0813556736593783, + 1.1376608658955782, + 1.0575655337513385, + 1.2464916954301395, + 1.250597854003308, + 1.3673005867960972, + 1.395239339300371, + 1.4969622785492618, + 1.637841740416305, + 1.5818249153818043, + 1.5526544214928784, + 1.9913289205912372, + 2.0087720723254225, + 1.967450414992637, + 2.008681400625713, + 1.9857905291239213, + 2.0191780158016037, + 2.0106882972279045, + 2.0103456948204546, + 2.012048288533708, + 2.040197244263803, + 2.007435386175821, + 2.0037367252676592, + 2.0173709153032657, + 1.9956652261706864, + 2.0358145653640225, + 2.0294436664090227, + 2.0225303174040015, + 1.9993764127351827, + 1.9938479954889328, + 2.0096192215622324, + 1.993914864108575, + 1.8966997528668426, + 2.0004430439571212, + 2.006726070638207, + 2.009253595137227, + 1.9686953235542308, + 1.9981397916392698, + 1.9921288605114214, + 1.970429393588553, + 2.0081861599689557, + 1.954608838368678, + 2.011792047261247, + 2.017549937061273, + 1.9400477704940098, + 2.024513791196103, + 1.9997581840545466, + 1.9954448634379904, + 2.024586727177093, + 2.0114467025699474, + 2.014137825557425, + 1.9954717345354567, + 1.9884206272660716, + 1.9971601308919755, + 2.0081960329052073, + 1.98583515552569, + 1.9812924423362386, + 2.027471577921622, + 2.0158832733097753, + 2.0597034488521295, + 1.9526960982765296, + 2.0091099685385174, + 2.0096450048859653, + 1.9462518145067413, + 1.9891334953808055, + 2.0068593339249823, + 1.9948748776291119, + 2.015412578140028, + 1.9990845243857809, + 2.0123230503935448, + 2.0151240733557545, + 1.9973251649153188, + 1.986619397497945, + 2.0198293366952083, + 2.036521202620932, + 2.014519923537227, + 2.0212941148670573, + 2.0273949772640876, + 2.0186735287823456, + 2.023478341755407, + 2.00900034240693, + 1.9437287173439408, + 1.97322229798543, + 1.9929718462263626, + 1.9873140943290544, + 1.9963443106174914, + 1.996694707613294, + 1.9451371611408712, + 2.007717035169656, + 1.9907004654565617, + 2.017370250085288, + 2.0259184659679352, + 2.0380736941164663, + 1.9692304761133617, + 2.0314439196323963, + 2.0122106790786862, + 2.0529897379244346, + 2.0189513309388705, + 2.01612253757418, + 2.036467161719555, + 2.031430876388372, + 2.008257019259746, + 1.9625641421755429, + 2.0249026778230803, + 1.9979376730583578, + 2.000583834417853, + 1.8701753530021357, + 1.9923297582423665, + 2.007727479621673, + 2.0177601912855874 + ], + "dft_magnitude": [ + 237.6620173124159, + 16.80665567799494, + 12.376704442961287, + 6.393773922901843, + 0.24856222519648202, + 5.432400840788968, + 7.065993457986646, + 6.848617442928084, + 5.946888525188848, + 3.9482631224300597, + 2.2375017610896957, + 1.2433162774056132, + 0.2489360033412261, + 1.6158823952878114, + 2.552855061006199, + 3.094510003173465, + 3.384343635676318, + 2.3693045671452513, + 1.211057923247144, + 0.6036372393529356, + 0.29993036843320886, + 0.7582065557948761, + 1.8371096751632998, + 2.7110271197294256, + 2.6007750211179843, + 2.3796115090088907, + 1.8037061689085263, + 0.6863060500074625, + 0.4960680933571733, + 0.602101269071251, + 1.0862541557534173, + 2.1306937669182826, + 2.097776870312458, + 2.580694301057285, + 1.8469620379640233, + 0.9502212494020588, + 0.3972473368538103, + 0.567706413132876, + 0.882496238686357, + 0.7263899549462309, + 0.3425224340740273, + 0.5096043479718544, + 0.5232103449130181, + 0.4504682463791083, + 0.35762063291154966, + 1.0503786246418583, + 1.5470137337868481, + 1.1834219841315499, + 1.5071370607454637, + 0.701918983605247, + 0.6585097032232187, + 0.3877774456123773, + 0.14556537914900833, + 0.794985009079022, + 0.957702044320269, + 0.8341671611436559, + 1.0664567304969783, + 0.9017117226266429, + 0.9767952409603209, + 1.281424441198275, + 0.8945765208091042, + 0.41013626554409377, + 0.3567356891094448, + 0.47066147597639146, + 0.7723559058495653 + ], + "mean_band_dot": [ + -2.6679754228098318e-05, + -3.614437140697646e-05, + 1.1241014590268605e-06, + 5.026633672855496e-06, + -1.0569628642542739e-05, + 1.058326219549599e-06, + -1.34802621118979e-05, + -1.458616104343946e-05, + -1.6593941808196178e-05, + -1.3901305123908969e-05, + -1.693324372809002e-05, + -1.5361182136075513e-05, + -2.793839911419127e-05, + -2.108469914219313e-05, + -2.2098403633208363e-05, + -2.088874612127256e-05, + -2.7481453173550108e-05, + -8.970661269813718e-06, + -6.281281343945011e-06, + -4.03528889592053e-06, + -8.70783233608563e-06, + -1.5657657712608852e-05, + -2.2802246917308366e-06, + -1.8904215437487437e-05, + -8.837077984935604e-06, + -1.2056114314873412e-05, + -2.3349125171989726e-05, + -1.781978573944798e-05, + -1.6330330822711403e-05, + -2.3010486756902537e-05, + -2.4075779037957545e-05, + -2.3868375478741655e-05, + -2.962122459848615e-05, + -2.2809125539424713e-05, + -2.1376339248035947e-05, + -2.202654366101342e-05, + -2.8252119022909028e-05, + -2.4202863187383628e-05, + -2.0302620754364398e-05, + -3.0716182891410426e-05, + -3.05798991746542e-05, + -1.9221608340558305e-05, + -2.0799247295144596e-05, + -1.9120375782222254e-05, + -2.6314920887671178e-05, + -1.976437079065363e-05, + -2.5931288533342922e-05, + -1.9403942303597432e-05, + -2.0338710129408355e-05, + -2.8104810212425946e-05, + -1.8517661544592556e-05, + -2.6021554617727816e-05, + -2.003344832246512e-05, + -2.2890038110290334e-05, + -1.86800355095329e-05, + -2.545822633237549e-05, + -1.8993645824139094e-05, + -2.348228332493818e-05, + -2.3873789359640796e-05, + -2.243111885036342e-05, + -2.8835911052738084e-05, + -1.6000345567590557e-05, + -1.9211220092074655e-05, + -2.1202094558248064e-05, + -2.4753061154569878e-05, + -2.830965684097464e-05, + -1.9068989900006272e-05, + -3.5979298871779974e-05, + -1.776636895556294e-05, + -2.1209450096648652e-05, + -2.6916168110346916e-05, + -2.8196539233249496e-05, + -1.6183202092179272e-05, + -2.6334485951906572e-05, + -2.726073819303565e-05, + -2.264837524990071e-05, + -2.2356573481374653e-05, + -2.6794304403665592e-05, + -2.4195176365537918e-05, + -2.5617342856776304e-05, + -2.649924013553573e-05, + -2.897792452927206e-05, + -1.8029148009190976e-05, + -2.1198294120949868e-05, + -3.070031067409218e-05, + -2.4515608174624504e-05, + -2.5651176997598668e-05, + -2.4696563173165487e-05, + -2.8083312543003558e-05, + -2.638357057094254e-05, + -1.9530572387793654e-05, + -1.822994295253011e-05, + -2.0513019876489125e-05, + -2.5082548773980307e-05, + -2.5628736182170542e-05, + -2.4283808301106546e-05, + -2.3057308226270834e-05, + -2.6225113970212988e-05, + -3.291402811100852e-05, + -2.4712044080388296e-05, + -2.2564717028217274e-05, + -3.274647508533235e-05, + -1.6450602004169923e-05, + -1.5039297750263358e-05, + -2.236472045069604e-05, + -2.723199096976714e-05, + -2.073495795684721e-05, + -2.0393768977555737e-05, + -2.4041631263571617e-05, + -2.039133633502388e-05, + -2.631008362641296e-05, + -1.435543390471139e-05, + -1.9755942389565462e-05, + -1.858197890669544e-05, + -2.8050938396972924e-05, + -1.9087827126895718e-05, + -2.9074408530505025e-05, + -1.9223395611334126e-05, + -2.2457888690041727e-05, + -1.6360918607460917e-05, + -2.4628476580801362e-05, + -1.835772275171621e-05, + -2.437162510204871e-05, + -2.4801670861052116e-05, + -2.3673021971148955e-05, + -1.6722583495720755e-05, + -2.7533449383554398e-05, + -2.2358329374583263e-05 + ] + }, + { + "layer": 3, + "mean_snr": 0.9069709737276288, + "per_head_snr": [ + 0.8886106572088394, + 0.8830813060837482, + 0.9392912393779795, + 0.8950363280067145, + 0.9113078757967848, + 0.9530236098996355, + 0.9149326230500969, + 0.8704841503972316 + ], + "mean_band_energy": [ + 1.14965517936038, + 1.1736100770408036, + 1.3702542067663002, + 1.5787498988481783, + 1.3403444508677245, + 1.4543076285738987, + 1.7411755141265814, + 2.1392877419552736, + 2.364230297194224, + 2.2136706279177787, + 2.51784474141345, + 1.7281204113393862, + 2.0485266778332285, + 2.581884609086421, + 2.603214635334105, + 2.682033093465888, + 1.017249209623957, + 1.0527922784833166, + 1.3799333654655195, + 1.4297023093383343, + 1.3427662837193353, + 1.8012573224603532, + 1.9002104630462888, + 2.0597107316500107, + 1.9006328962074264, + 1.8750358014636488, + 2.520437921715031, + 2.325398698023168, + 2.44209834845387, + 2.659974726798078, + 2.5470905129770074, + 2.67525373515471, + 2.597714638166119, + 2.673505838513094, + 2.6417724185206826, + 2.624539561909545, + 2.5638848767910467, + 2.6648458080145456, + 2.667059606400807, + 2.6521987285825244, + 2.6611929674009644, + 2.6690802899523973, + 2.5947215793862526, + 2.652306726986355, + 2.6314192635215132, + 2.5898155832521716, + 2.6474845728826573, + 2.6375315582652075, + 2.629667911668763, + 2.6224825370573406, + 2.6493865639254093, + 2.665164698276783, + 2.6567960183568964, + 2.674912909195294, + 2.667582340169453, + 2.6802162458455694, + 2.6585479903264133, + 2.679223004294757, + 2.6219877726132914, + 2.6602396164835476, + 2.6656847929257275, + 2.658161646697639, + 2.6167317841229085, + 2.654155014502603, + 2.676155680317395, + 2.6324596546837338, + 2.65532978841183, + 2.6409845871404833, + 2.581897073301869, + 2.6758813030370785, + 2.668258425896447, + 2.649016189879614, + 2.658835636558768, + 2.6185563206198665, + 2.665480607238589, + 2.628551120824545, + 2.615751681134503, + 2.638338173486243, + 2.5617669424556624, + 2.6592412862763215, + 2.680415199006383, + 2.6344985820970166, + 2.691127220632165, + 2.6661253149849387, + 2.683575993021311, + 2.6144574164933534, + 2.614168030566047, + 2.6647450311624468, + 2.664004033817939, + 2.68129939253639, + 2.6885870158479985, + 2.631463719626298, + 2.6604860143700684, + 2.652932363990738, + 2.6706148643296945, + 2.6347857826855385, + 2.6769261976892498, + 2.6752207307551465, + 2.683282091077931, + 2.635544890982368, + 2.6523640149812806, + 2.614569377501512, + 2.6127433802107447, + 2.632015159707519, + 2.6513040603824187, + 2.6039091718433056, + 2.6600667286277613, + 2.6092040139363597, + 2.6761341886684757, + 2.675044053274476, + 2.639048078327715, + 2.6217168672082884, + 2.6417968782764305, + 2.596417108326687, + 2.513951558766312, + 2.639903076384856, + 2.6234912211060646, + 2.636470753068373, + 2.6532080159588762, + 2.6826114042156384, + 2.636033864692246, + 2.6657789845578783, + 2.6457084080282236, + 2.648149629767376, + 2.6758379904344256, + 2.630598579963369, + 2.6422965834386254, + 2.6413970709977868 + ], + "dft_magnitude": [ + 315.499003840301, + 21.300242358080414, + 15.423434591917193, + 7.84906613746902, + 1.2118980956061036, + 5.457171976005134, + 9.185305007090886, + 10.078448624928088, + 9.08441473093367, + 7.201465389898711, + 4.058127114782623, + 1.9942067653271882, + 1.0223338388490109, + 1.2947193024958434, + 3.098266939984505, + 4.489853542811581, + 4.326132629698235, + 3.871186681544906, + 2.3375731549654666, + 1.9452813590300824, + 2.365132384421192, + 3.2443318354676345, + 3.4408794055733347, + 3.805384451160101, + 3.541295860481417, + 3.4836805577030305, + 2.8660707039645645, + 2.5245471215247335, + 0.8270250851383583, + 0.4683823384227067, + 1.6421641910825977, + 2.816795281998302, + 3.3726401097864995, + 3.552624241230438, + 2.8774991654600552, + 1.5202824375282766, + 0.22441464540089584, + 0.7606694054799766, + 1.1879510542274312, + 0.8528727001577263, + 0.9970373942081493, + 0.7479287458605308, + 0.8927122619473808, + 0.6705570110058977, + 0.17628455543743046, + 1.7114507255411893, + 2.5365092820843, + 2.563149124476567, + 2.578091189226153, + 1.5202452933865733, + 1.2983523290398231, + 0.2024794560376958, + 0.5365455980486981, + 0.3310242658567112, + 1.4404098268369672, + 2.197913382276364, + 2.4266069142971456, + 2.310645508038224, + 1.6568324364778688, + 0.8377171583232672, + 0.4916145688129261, + 0.2852828598129031, + 0.8184886116598996, + 1.2338535582079404, + 1.3431093025873793 + ], + "mean_band_dot": [ + -1.6581177419539017e-05, + -1.2983858027837414e-05, + 2.45247636030399e-05, + 4.581331211284123e-06, + 1.0233261264147586e-05, + 1.3615606803796254e-05, + -8.532177773190597e-06, + -3.847375086252214e-05, + -4.432586888469814e-05, + -6.564196200997685e-05, + -3.074525017154883e-05, + -3.116536225888922e-05, + -5.284180690523499e-05, + -5.099142364883846e-05, + -5.416150270320941e-05, + -3.755895704671275e-05, + -3.959380324403128e-05, + -2.706210867131631e-05, + 8.347497498562007e-06, + 1.8149775087294984e-05, + 1.3807311574964842e-05, + -4.84793373800585e-06, + -9.9912407733882e-06, + -1.8038229882222367e-05, + -3.998639726887632e-05, + -1.915553468734288e-05, + -5.849818063552448e-05, + -2.838806756244594e-05, + -4.823944888698861e-05, + -5.1927726872236235e-05, + -5.782740441873102e-05, + -4.242522754793754e-05, + -5.707224045181647e-05, + -5.0660560361848184e-05, + -5.2916194476893e-05, + -5.176369529635849e-05, + -6.0298259541013977e-05, + -4.918140172094354e-05, + -4.195594323164187e-05, + -4.093335979860058e-05, + -5.239053911054725e-05, + -4.716788146197359e-05, + -4.4790535412175814e-05, + -6.025441314250202e-05, + -4.200763549988551e-05, + -5.126810572164686e-05, + -4.0609736913665984e-05, + -4.566643389125602e-05, + -4.629744375961309e-05, + -4.671634860642371e-05, + -5.480709069161094e-05, + -4.778113310521803e-05, + -3.656448043898308e-05, + -5.098134727177239e-05, + -4.726959031131628e-05, + -4.838410177399055e-05, + -3.477413486052683e-05, + -4.411719385188917e-05, + -5.176860588562704e-05, + -4.392709718104015e-05, + -4.3179312910979206e-05, + -4.688535017294271e-05, + -3.932672188966535e-05, + -4.8280033752234885e-05, + -4.949129629494564e-05, + -5.3201360472598935e-05, + -4.087776426331402e-05, + -4.198522069032151e-05, + -5.342467193258926e-05, + -4.6950618980190484e-05, + -4.7569304342687246e-05, + -5.420871025307861e-05, + -3.838696756019999e-05, + -4.6542034056074044e-05, + -4.569877910398645e-05, + -5.711925427931419e-05, + -5.3230345542942814e-05, + -4.494946085742413e-05, + -4.388092571616653e-05, + -4.493141477723839e-05, + -5.212918136976441e-05, + -4.8007249233705807e-05, + -4.7291228838730603e-05, + -4.479697309989206e-05, + -4.9753486223380605e-05, + -5.182911365864129e-05, + -4.544851537957584e-05, + -4.160790899732092e-05, + -4.771368264755438e-05, + -3.821192194664036e-05, + -4.887425438937498e-05, + -6.149466599936204e-05, + -4.5404242456470456e-05, + -4.411867033127237e-05, + -5.042718848358163e-05, + -4.3353963974368526e-05, + -4.903021238078509e-05, + -5.852607398537657e-05, + -4.6624528948768784e-05, + -4.351156354687191e-05, + -5.2341024115776236e-05, + -4.946872650180012e-05, + -5.576155342623679e-05, + -4.6566120261104516e-05, + -5.420526258603786e-05, + -5.087999238639895e-05, + -4.7876479584374465e-05, + -4.3968586282971955e-05, + -4.747608068100817e-05, + -4.7364798945181974e-05, + -4.9569434963814274e-05, + -4.4433787422804016e-05, + -4.962248509343681e-05, + -4.044888919452205e-05, + -4.911955880970709e-05, + -4.616089731257489e-05, + -4.5707711706199916e-05, + -4.103684273104591e-05, + -4.7853949070031376e-05, + -5.376665467338171e-05, + -5.153205177066411e-05, + -5.445638885248627e-05, + -4.9504319576954003e-05, + -5.194477427039601e-05, + -4.737365838991536e-05, + -4.479528580247916e-05, + -5.149809564386487e-05, + -4.794737640168023e-05 + ] + }, + { + "layer": 4, + "mean_snr": 0.8816683556250567, + "per_head_snr": [ + 0.8336422505380886, + 0.9113558096912876, + 0.9377648398144715, + 0.8835751752558973, + 0.7798024315976192, + 0.8932823813185979, + 0.88101709331209, + 0.9329068634724018 + ], + "mean_band_energy": [ + 0.8215158070314099, + 0.8433512824383484, + 1.0995636271172962, + 1.3249931818064136, + 1.227265011430387, + 1.3639225776390074, + 1.4670202756203352, + 1.6523233539628768, + 1.63692621462065, + 1.7381160953433916, + 1.8664962298828804, + 1.836646124050887, + 2.1748128979186987, + 2.3929605417678097, + 2.628683861284552, + 2.3030576891823014, + 0.885542262811601, + 0.9700708931025463, + 1.0627775154057901, + 1.3506839275020708, + 1.2925081460807033, + 1.3778761759839098, + 1.5337591992183714, + 1.554420472103116, + 1.6917473984756835, + 1.6340359041417454, + 1.835390330330096, + 1.933058309586647, + 2.1378162486807666, + 2.473091262193728, + 2.701561887455002, + 2.619971404164321, + 2.748751835502903, + 2.756764097340909, + 2.715506358806403, + 2.561069875647611, + 2.71598785432921, + 2.7670101721947713, + 2.748096022557318, + 2.619928031084143, + 2.734560451255698, + 2.713005957085926, + 2.556728031464269, + 2.6427053525395703, + 2.756326256190894, + 2.638870222350855, + 2.75063778533867, + 2.57520506208337, + 2.6069944516088803, + 2.5685500595790796, + 2.592950214483464, + 2.5685386938648094, + 2.585519184014707, + 2.7312684663140914, + 2.764486683456873, + 2.7362260941832126, + 2.5161928608710813, + 2.582795485902425, + 2.667403009037569, + 2.583230768223771, + 2.7188606676573492, + 2.765950486771052, + 2.724347212755939, + 2.6431461913189302, + 2.7657759512629756, + 2.7371078089590632, + 2.745092154725624, + 2.6949499984876564, + 2.7830825942179374, + 2.7178747689712566, + 2.771663800025646, + 2.7585398039469258, + 2.7042571891037728, + 2.6100895216066373, + 2.7304043909511506, + 2.642015823203714, + 2.4240562846332026, + 2.5503972328129603, + 2.558280175850115, + 2.641774605667365, + 2.7603120560322143, + 2.7318000581657174, + 2.769611989369494, + 2.7507819826804987, + 2.734384555129264, + 2.512169694445264, + 2.4672549892136146, + 2.7039943107189206, + 2.6009187123425868, + 2.739753032544426, + 2.7095487840191588, + 2.7094365862790877, + 2.66851505832466, + 2.748509780662685, + 2.512385851562417, + 2.6128776010011325, + 2.6259854021364433, + 2.6863626347724168, + 2.730305749949393, + 2.7423969730216573, + 2.7120992729593496, + 2.7276620365817905, + 2.747407103492325, + 2.7933253770835584, + 2.757196002591975, + 2.6926182251686885, + 2.7849262349234536, + 2.740024775919311, + 2.7310629332319802, + 2.475488138211743, + 2.7243612483588624, + 2.584597347661669, + 2.4279622836458987, + 2.8009685934337174, + 2.6228416129367513, + 2.5770369473013464, + 2.754457090009696, + 2.7164731700456315, + 2.530442211947774, + 2.630076874149408, + 2.560808701266051, + 2.6935211933361263, + 2.779632771361074, + 2.5828619881836214, + 2.485785235811676, + 2.7480195957344318, + 2.4300520739109466, + 2.6665708377386945 + ], + "dft_magnitude": [ + 309.62052778794373, + 29.567946600473025, + 21.189456738458347, + 10.225192509743348, + 1.9847817726412886, + 8.131968057627386, + 10.989014117147862, + 12.430401018275596, + 7.6194484394539534, + 5.608643901731712, + 2.5664222595849058, + 3.3380196823163297, + 1.0950647289781326, + 3.1003148867523596, + 4.022361011174439, + 6.924272917725005, + 5.204250680378226, + 4.235415249978126, + 2.3341124789376453, + 2.016749405301892, + 0.3571975816282354, + 0.7436404254488799, + 2.3857750410730345, + 3.6162959000303783, + 3.563162839048326, + 4.016369671288263, + 3.3007236219915845, + 2.244488857744067, + 0.18772081775809094, + 0.7954453904185969, + 2.8381309092201685, + 1.8002547030935474, + 2.8181113808830336, + 2.8672337032359847, + 1.6907962732740998, + 1.3871071482512467, + 0.4406177070878096, + 1.6718401171323507, + 0.951523056316458, + 2.2721150358269724, + 1.0883359852657033, + 1.6063574661602513, + 0.23768412944730088, + 1.3064901779953917, + 1.780069866034287, + 0.5475494145867629, + 1.7978672590482534, + 1.367967595098395, + 2.1068735936589955, + 2.08719702722683, + 1.048734109646097, + 1.3400001539311546, + 1.0516617307740124, + 0.969485187099764, + 1.4651424742948491, + 1.7199796052122005, + 1.4721286975789638, + 2.2412567712233185, + 0.9994165334676104, + 1.1379212511275831, + 1.2416616492864916, + 1.6089176774942284, + 1.53396759260863, + 2.178225529042397, + 1.4653152719578486 + ], + "mean_band_dot": [ + -8.868827130470436e-05, + -9.166804284177488e-05, + -7.040858622531232e-05, + -7.312307394613526e-05, + -4.166950017747695e-05, + -2.95894486441739e-05, + -3.0337671887537e-05, + -4.348366053363861e-05, + -1.0388823397988745e-05, + -4.029772031799439e-05, + -3.921216867297517e-05, + 1.2985384046260151e-05, + 4.2392505179122963e-05, + 6.312542541309085e-05, + 8.310811540468421e-05, + 5.828903029225785e-05, + -9.220424487921264e-05, + -0.00011436918975959998, + -6.444899344160149e-05, + -8.740485911573614e-05, + -4.979946038474736e-05, + -5.1096751235490956e-05, + -3.7218898256696775e-05, + -2.698865881711754e-05, + -2.4534252020202985e-05, + -3.232409886777532e-05, + -1.648923816333081e-05, + -1.1064847626585106e-05, + 1.0851964930225222e-05, + 7.690819927574921e-05, + 9.173177181764913e-05, + 9.910147036862327e-05, + 0.00011601951814554923, + 0.0001060854124261823, + 9.692093965441018e-05, + 8.427098680385825e-05, + 0.00010413040337198254, + 0.00010244990721730574, + 0.0001015189166082564, + 7.053513354549068e-05, + 0.00011049641580029856, + 0.00010557922405496356, + 8.705528478003544e-05, + 0.000108182227791076, + 0.00011438806450314587, + 9.323590484200395e-05, + 0.00010335849128750851, + 8.112983414321207e-05, + 7.716431798598933e-05, + 0.00010332638112231507, + 8.890483718460018e-05, + 9.161378397948283e-05, + 9.764370292941749e-05, + 0.00010759118856640271, + 0.00011369814859563121, + 9.95276054709393e-05, + 7.786311743984697e-05, + 9.928500981004618e-05, + 0.00012072615049874003, + 9.411183623342367e-05, + 0.00010780720560887858, + 0.0001066437475856219, + 9.854305454837231e-05, + 9.304213494942815e-05, + 0.00010908852777902212, + 0.00010446728299484676, + 0.00010265991414826203, + 0.00011578914734400314, + 0.00010911334345564683, + 0.00011239198249768378, + 0.00010399416669315542, + 0.00012002376411146543, + 0.00011090289262938313, + 9.77108220467926e-05, + 0.00010583189032331575, + 8.884266480890801e-05, + 7.648169707863417e-05, + 8.335490747413132e-05, + 9.118093271354155e-05, + 9.494832352174853e-05, + 0.00010927966059171013, + 0.00010516992193743135, + 0.00010716597557802743, + 0.00012025194814668794, + 0.00010581929359432252, + 8.089291480928296e-05, + 8.996594942800584e-05, + 0.00011577015686725645, + 0.0001016944150933341, + 0.00010532682654229575, + 0.00010582132995295979, + 0.00010622771242196904, + 0.00010278714506739561, + 0.00010770061157927557, + 7.56896542384311e-05, + 9.831703459894925e-05, + 8.956909823609749e-05, + 0.00010261441423153883, + 0.00010461344186296628, + 0.00010909724335306237, + 9.960543650322506e-05, + 0.00010902897690812097, + 0.00010906824945777771, + 0.00011326124251809233, + 0.00010752178445727623, + 9.971374413453304e-05, + 0.00011042130813621043, + 0.00010814342510911956, + 0.00010386722300381734, + 9.540525235252062e-05, + 0.00010207317200183752, + 8.857554189489747e-05, + 7.306441011678544e-05, + 0.00010705078489081643, + 0.00010056901271582319, + 8.73160931860184e-05, + 0.00010058878069685306, + 0.00011122950240860519, + 8.879624408564268e-05, + 9.17073882646946e-05, + 8.072098887623724e-05, + 0.0001017800719864681, + 0.00010804898471405977, + 8.268335284355999e-05, + 7.257759580170386e-05, + 0.00010514022119423316, + 8.577003967502606e-05, + 0.00012273888114577858 + ] + }, + { + "layer": 5, + "mean_snr": 0.8769781560117472, + "per_head_snr": [ + 0.9274908536465192, + 0.851502535055023, + 0.8840202405729911, + 0.9254577545491846, + 0.8521626874132568, + 0.9118107800306798, + 0.8214119660366277, + 0.8419684307896957 + ], + "mean_band_energy": [ + 0.7039898942387248, + 0.8113764706787019, + 0.9095487371544944, + 1.1012991484485832, + 1.1020555691267329, + 1.302905322319543, + 1.4355262545811485, + 1.8659406424294307, + 1.723225921612805, + 1.9660414236063275, + 2.0623332311065248, + 2.4368064163863234, + 2.0960067249791123, + 2.541126019898451, + 2.5707256277547987, + 2.7161148664757384, + 0.706264976378479, + 0.752915964256399, + 0.9350537254922973, + 1.1195285983016494, + 1.0929933746488025, + 1.2914566079034344, + 1.5538792287731447, + 1.7067349451894742, + 1.658328777484476, + 1.9131260019313885, + 1.9364409763128645, + 2.364066017258967, + 2.2124826856365285, + 2.4596704917088044, + 2.533753474160612, + 2.6874442827544307, + 2.7225524046474288, + 2.7397343998112564, + 2.68021757750647, + 2.7301538507999776, + 2.6742242686263378, + 2.7151549548196723, + 2.744434650329989, + 2.7223619179482554, + 2.697174943319256, + 2.7198013122030584, + 2.623143211610053, + 2.7235837523946476, + 2.714298893829678, + 2.5695711902475527, + 2.6929673962958542, + 2.7027417149232953, + 2.469017673823638, + 2.7587630734083213, + 2.389059542583171, + 2.7271964150980548, + 2.7075965303199307, + 2.6959043130710274, + 2.6698949284878886, + 2.7029857239530966, + 2.7369621732614178, + 2.7044261730262296, + 2.571457437033226, + 2.7116916150134394, + 2.544557474479932, + 2.480817766074205, + 2.6742086623063415, + 2.7509917402241566, + 2.502917832596717, + 2.7226095167487294, + 2.72040702323792, + 2.6179573699556205, + 2.7006426594948145, + 2.74174909253785, + 2.5043302693917155, + 2.686946496067158, + 2.722098320028218, + 2.6861284380656905, + 2.725252985586824, + 2.6257360832232077, + 2.682979057857187, + 2.699507998271379, + 2.7445388848074703, + 2.658027589284826, + 2.734583496819198, + 2.683699477409114, + 2.694550045395239, + 2.45437108767727, + 2.7067288329854335, + 2.71911127635693, + 2.674340398009525, + 2.7451205446344176, + 2.598130391589738, + 2.703390761780655, + 2.585448075287081, + 2.734869737856423, + 2.700450306888511, + 2.695634131244791, + 2.7247860605962444, + 2.692653810639568, + 2.706390374711674, + 2.716250777948872, + 2.5200680385277945, + 2.524247554971595, + 2.7120851516872957, + 2.5995513924212723, + 2.6762921286941648, + 2.6715388840547565, + 2.5564784883884943, + 2.5451022194360338, + 2.7304368071915324, + 2.7340096020217963, + 2.5806700926328308, + 2.7416176558045695, + 2.6954389545475053, + 2.376349490930126, + 2.7261264765128423, + 2.7203432688770555, + 2.6091790617632054, + 2.7347529392699332, + 2.7167732922560806, + 2.719062845393845, + 2.6992668395319157, + 2.6882793799807043, + 2.6707534930263535, + 2.72016285532794, + 2.7051069135016466, + 2.734043112122416, + 2.601638774678614, + 2.6984190271222275, + 2.7404407216512965, + 2.7123970722893525 + ], + "dft_magnitude": [ + 310.60978182006727, + 27.78285991648857, + 20.290925343278527, + 11.435225015344681, + 0.20000535677441353, + 8.85300778619559, + 13.557980702172312, + 12.864862677813823, + 11.703716652021848, + 9.012284247746285, + 4.890716028427091, + 3.4849236183811114, + 0.996575831332105, + 2.945126143335853, + 4.459956107253261, + 5.643189351553669, + 5.908254212291177, + 4.186912123358142, + 2.505899565487913, + 2.439225887906, + 0.16264922244139415, + 1.7688328660072128, + 3.634106104423438, + 3.285627429383506, + 3.35466076268222, + 2.9768620407155484, + 3.135275146663013, + 1.20443356257175, + 0.3191863933461952, + 1.7305809820381348, + 2.7728364627375277, + 4.57584215130852, + 3.661182663059758, + 3.201834315227856, + 2.5893071900070717, + 1.7279234123928926, + 0.6530011636978558, + 0.6711857330589321, + 2.8165553608716354, + 1.768816596127329, + 2.753236095277675, + 1.6788624855155128, + 1.8319622492578735, + 1.415264195614892, + 1.334245837101297, + 1.354761090225186, + 2.5586148403251174, + 0.346976123631668, + 2.069614572107149, + 1.3942322630676462, + 1.7116586836175867, + 1.6902943429049049, + 0.31862144679674503, + 1.5140485368916279, + 2.262843507082957, + 1.3892490321271371, + 1.5778982679947817, + 2.155287801031476, + 0.32434573761801144, + 0.7042704061231184, + 0.4416158213047453, + 1.960475463196569, + 2.063707931296015, + 3.344357747310281, + 4.782367424512785 + ], + "mean_band_dot": [ + -6.534829140036891e-05, + -8.17787849882734e-05, + -4.860020453634206e-05, + -4.5830516341993643e-05, + -1.8454074648843743e-05, + -1.634393510130394e-05, + -1.5950148338106374e-05, + -4.32577061246775e-05, + -2.6221357757094665e-05, + -2.3810897161524736e-05, + -2.3507110370246664e-05, + -1.2672929642576491e-05, + 3.2112601502376492e-06, + 3.8066405011250026e-05, + 5.587542818830116e-05, + 6.765670764252718e-05, + -7.244114158311277e-05, + -8.005473546290887e-05, + -4.490937106993442e-05, + -3.6942734197964455e-05, + -1.9614953743030128e-05, + -2.257992156273758e-05, + -1.4435288335334917e-05, + -2.5700669084471883e-05, + -2.889318716370326e-05, + -5.867139157089696e-05, + -3.4522705846029567e-05, + -1.0875413408939494e-05, + 3.22090220379323e-05, + 4.699911869465723e-05, + 4.521945129454252e-05, + 7.418529685310205e-05, + 8.688281013746746e-05, + 7.842151535442099e-05, + 7.203785889942083e-05, + 7.847580263842246e-05, + 5.8796718576559215e-05, + 7.685723812755896e-05, + 7.276894439200987e-05, + 7.633999484824017e-05, + 8.367727588165508e-05, + 8.708832137926947e-05, + 6.463726685979054e-05, + 6.66677101435198e-05, + 7.589928100060206e-05, + 6.734977068845183e-05, + 6.997405171205173e-05, + 7.182833292063151e-05, + 5.980314472253667e-05, + 8.640076748633874e-05, + 6.002270515637065e-05, + 7.405383757941308e-05, + 7.218967448352487e-05, + 7.632310655480978e-05, + 8.194991323762224e-05, + 6.96510824127472e-05, + 7.597246258228552e-05, + 9.12254204195051e-05, + 5.1759153393504675e-05, + 6.164574301692483e-05, + 5.184034080230049e-05, + 6.378726732236828e-05, + 6.575277348019881e-05, + 8.813322847345262e-05, + 4.86820704281854e-05, + 8.757063937991916e-05, + 8.02908302830474e-05, + 7.122844692730723e-05, + 6.618026964133605e-05, + 6.711254582114634e-05, + 5.600439794761769e-05, + 6.655115794274025e-05, + 7.300313427549554e-05, + 6.990135580053902e-05, + 6.777403859814513e-05, + 6.946838743715489e-05, + 6.110094091127394e-05, + 7.623055034855497e-05, + 8.659533250465756e-05, + 5.815894473926164e-05, + 6.20454366071499e-05, + 6.0656582263618475e-05, + 7.084744856911129e-05, + 2.7532605372471153e-05, + 8.125650992951705e-05, + 8.620340418019623e-05, + 6.623001786465466e-05, + 7.696718603256159e-05, + 4.892254492006032e-05, + 7.378128384516458e-05, + 3.288088987574156e-05, + 8.100580134851043e-05, + 7.355101570283296e-05, + 7.510829527745955e-05, + 8.488429261888086e-05, + 7.143930270103738e-05, + 6.397276592906564e-05, + 8.435203744738828e-05, + 5.321750950315618e-05, + 6.869525168440305e-05, + 7.026600997050991e-05, + 6.096398556110216e-05, + 6.787512393202633e-05, + 6.12338612882013e-05, + 5.4521221045433776e-05, + 4.642058706849639e-05, + 7.859495190132293e-05, + 7.595153283546097e-05, + 6.558853499427642e-05, + 8.378661323149572e-05, + 7.231978702293418e-05, + 1.3004551306039502e-05, + 7.05944971741701e-05, + 7.438905458911904e-05, + 4.727140685645281e-05, + 7.801532456142013e-05, + 7.505841244892508e-05, + 8.883715264573766e-05, + 7.480454678443493e-05, + 7.18213368600118e-05, + 6.851490388726234e-05, + 6.483088918685098e-05, + 6.535089596582111e-05, + 7.204178359643265e-05, + 5.416471549324342e-05, + 7.40919108466187e-05, + 8.56966926221503e-05, + 6.778355282222037e-05 + ] + }, + { + "layer": 6, + "mean_snr": 0.9063029050638602, + "per_head_snr": [ + 0.9187198937219253, + 0.8909582089699201, + 0.8890139325536887, + 0.9486000956776887, + 0.9008131663423035, + 0.9493962220103959, + 0.9031837967227531, + 0.8497379245122066 + ], + "mean_band_energy": [ + 0.8152468963666721, + 0.8701629137043758, + 1.0582118431400227, + 1.3273970978978369, + 1.3515103394294101, + 1.5421788089952955, + 1.7870165141567433, + 1.9765755594048109, + 2.061936111708219, + 2.1102369562718177, + 2.244414064355265, + 2.3415575495489467, + 2.338259764787063, + 2.3227253503697227, + 2.3807166834183775, + 2.4507289308889746, + 0.829927780482953, + 0.8208001428992171, + 1.0653143517695938, + 1.32871102643257, + 1.33980072799989, + 1.5538193237068976, + 1.8032023358428244, + 1.985047056247506, + 1.9168397506827493, + 2.1265747706667533, + 2.1794137797366506, + 2.425773183580935, + 2.3623141806453907, + 2.4246465245813766, + 2.4890596279757204, + 2.50828814082943, + 2.5438465330778968, + 2.5143151304619966, + 2.504422179934311, + 2.5172801361518324, + 2.521529117221087, + 2.4568242619685456, + 2.495919919297245, + 2.3276702838601526, + 2.4282598109050735, + 2.491030785248615, + 2.494338228407514, + 2.483406388031561, + 2.502492857802979, + 2.4922673968694493, + 2.50226848536078, + 2.5258732007293396, + 2.498790780576323, + 2.3906797486009825, + 2.532298397724059, + 2.4547342161789967, + 2.5299536245875363, + 2.5184642902885237, + 2.4976994217475994, + 2.3830443109769153, + 2.501362458437761, + 2.4949350073438428, + 2.515224652007939, + 2.5199938510706694, + 2.5162903125950127, + 2.5122893839208498, + 2.474426153155484, + 2.3613140582547345, + 2.5265550840467235, + 2.341874154518367, + 2.2555948309245313, + 2.487983386285249, + 2.5221695772809607, + 2.515218090607137, + 2.444862387085685, + 2.3846812203297354, + 2.5050167145094147, + 2.2792946701401, + 2.5071464900491414, + 2.487026537911145, + 2.368545793349652, + 2.4971818727845196, + 2.396549821487257, + 2.4731431963184995, + 2.5312252756382687, + 2.504667414125702, + 2.5124047022149014, + 2.529375697142278, + 2.4870548478390244, + 2.484778821757663, + 2.5375274848486704, + 2.4962608611489863, + 2.499979656256791, + 2.489806542034243, + 2.4969156375679393, + 2.4908992966553463, + 2.5129457783960163, + 2.505568821366202, + 2.5209134086176554, + 2.527542010889011, + 2.3744885164890537, + 2.526148108515427, + 2.504551296487195, + 2.5278598993864936, + 2.5105137559843786, + 2.481334618377476, + 2.512731215621317, + 2.500992482062369, + 2.509797679103448, + 2.504335797876344, + 2.511638457109747, + 2.4960123586599234, + 2.4889285455167496, + 2.503348123841892, + 2.465940206606218, + 2.518076489504919, + 2.5257398154827406, + 2.506834050043956, + 2.513036490234361, + 2.5026425096352973, + 2.530651810603679, + 2.508486234082561, + 2.4870861800675463, + 2.5102433517235365, + 2.3268308238854285, + 2.3355466171165045, + 2.4907615404139474, + 2.5095842079465545, + 2.511566119553457, + 2.476780590644635, + 2.482705766578153, + 2.5056556982748894 + ], + "dft_magnitude": [ + 296.4232129128767, + 19.146660558200654, + 15.27830105890529, + 6.920543610902991, + 0.5099321678329976, + 6.198418247094023, + 9.821635174898326, + 11.203428066868598, + 10.15176021986334, + 7.973828951810882, + 5.796557069001006, + 3.827451316660694, + 0.7450816693735566, + 2.0303631011408245, + 4.472985301766812, + 4.346148897069341, + 5.322365890193082, + 4.063811637520195, + 2.7383727138726774, + 1.1631783618768643, + 0.6574007261790747, + 1.8847015966399243, + 1.7336439112028161, + 3.068826502737023, + 2.959121531141366, + 1.9663618442146753, + 2.2112768412803163, + 1.6318215090383754, + 0.3652955897533243, + 1.4236682606640012, + 2.018142531535576, + 3.348556489701372, + 3.337890146432763, + 3.072216607613719, + 2.1473705482686705, + 0.9225510186569612, + 0.418211441495807, + 1.5614613351428348, + 1.045777885048823, + 2.2868180118953307, + 1.630375150898338, + 1.6195052641839907, + 0.8326943720722089, + 1.3517044478875309, + 0.2652143519720392, + 0.8834186998960746, + 0.5740346933196909, + 1.9735603474498713, + 1.1112722363895757, + 2.3675404855599345, + 0.952433829877904, + 1.2452910072699288, + 1.0354563149272018, + 0.6939811923265371, + 1.4159251930635675, + 1.4099043974409458, + 1.9920290952576267, + 0.8554097681959131, + 1.1495753298135971, + 0.934182470797921, + 0.40413572954425697, + 0.4145854586825654, + 1.368225918216335, + 2.2113457165217536, + 1.5138461225042477 + ], + "mean_band_dot": [ + -8.062182229195969e-05, + -8.880011409928557e-05, + -5.163071284641774e-05, + -5.010628592572175e-05, + -3.3805703310463286e-05, + -2.1741424234278384e-05, + -4.734528738481458e-05, + -6.26453534096072e-05, + -6.595121089958411e-05, + -6.063170997094858e-05, + -6.064494087354433e-05, + -2.563308180469903e-05, + -2.8406539058778435e-05, + -2.219993575636181e-05, + -2.3292580408451613e-05, + 1.431963511322465e-06, + -8.817268837901793e-05, + -8.380323174606019e-05, + -4.694967662999261e-05, + -4.663881558997218e-05, + -3.278669569795056e-05, + -4.708746345727377e-05, + -4.104153651951492e-05, + -6.639585032530704e-05, + -6.280339857767103e-05, + -6.807241504702688e-05, + -4.601439627549553e-05, + -1.1766530178647372e-05, + -4.560825868793472e-05, + -3.146412001342469e-05, + -2.3584479777127854e-05, + -1.2164777444922947e-05, + -1.4986952010076493e-05, + -1.7991411823459202e-05, + -2.5258883397327736e-05, + -1.5172861594692222e-05, + -2.4966650585156458e-05, + -1.5316459894165746e-05, + -2.292506371759373e-05, + -2.303059500263771e-05, + -1.1480820376164047e-05, + -2.6721563017417793e-05, + -1.3450315691443393e-05, + -1.8555568203737494e-05, + -3.1994884011510294e-05, + -1.1534011264302535e-05, + -2.216342386418546e-05, + -8.260852155217435e-06, + -3.145534446957754e-05, + -2.2007680399838137e-05, + -2.690056589926826e-05, + -2.117735130013898e-05, + -2.653146225384262e-06, + -1.8748528646028717e-05, + -1.4700413998980366e-05, + -2.852140096365474e-05, + -2.142190191989357e-05, + -2.145739887282616e-05, + -2.4773726124749373e-05, + -1.9835665852951934e-05, + -2.8998831112403423e-05, + -2.1450963799907186e-05, + -1.7413182376913028e-05, + -3.508812437758024e-05, + -2.4124211449816357e-05, + -2.8222996206750395e-05, + -2.942620369594806e-05, + -1.899650973768985e-05, + -1.980253682631883e-05, + -2.136518287443323e-05, + -1.70046927223666e-05, + -2.7420799824540154e-05, + -2.5844764422799926e-05, + -3.1226785836224735e-05, + -1.8427023405820364e-05, + -2.515451549811587e-05, + -2.117656387667921e-05, + -2.719284088925633e-05, + -2.064686100311519e-05, + -9.646483704273123e-06, + -9.255365057470044e-06, + -2.4306063323820126e-05, + -6.6631863546717796e-06, + -1.8983371319336584e-05, + -1.6212723949138308e-05, + -2.4209537286878913e-05, + -1.5698507695560693e-05, + -2.3677891704210197e-05, + -1.2808386145479744e-05, + -1.4796103641856462e-05, + -1.7624753525069536e-05, + -2.121101698548955e-05, + -1.864395426309784e-05, + -2.0046729787281947e-05, + -2.4202280883400817e-05, + -1.829708617151482e-05, + -2.746458699220966e-05, + -1.8628177372193022e-05, + -3.3731251505741966e-05, + -1.6241394405369647e-05, + -2.0204554402880603e-05, + -8.519947527929617e-06, + -2.646902680680796e-05, + -1.5435748537129257e-05, + -1.868869685495156e-05, + -1.786705558970425e-05, + -1.5730808172520483e-05, + -1.4733113061993208e-05, + -2.6963142772729043e-05, + -2.465138777552056e-05, + -1.2516126389527926e-05, + -1.1053321031795349e-05, + -1.8941446342068957e-05, + -1.8168614531077765e-05, + -1.5111026357317314e-05, + -1.8317894273423008e-05, + -3.191358177900838e-05, + -3.5614687021734426e-06, + -1.7527989996324322e-05, + -2.4826499611663166e-05, + -3.0264060114859603e-05, + -2.2438210180553142e-05, + -1.6171563174793846e-05, + -2.2305479205897427e-05, + -1.9665791938905386e-05, + -1.6473248706461163e-05, + -2.214554251622758e-05, + -2.382981210757862e-05 + ] + }, + { + "layer": 7, + "mean_snr": 0.8961294227068274, + "per_head_snr": [ + 0.84549300139498, + 0.8195190576058302, + 0.8378574858252286, + 0.929510244335315, + 0.948222539484443, + 0.9331893454068907, + 0.9448321448277868, + 0.9104115627741444 + ], + "mean_band_energy": [ + 0.6231627549878631, + 0.6151271365174824, + 0.9377573002393227, + 1.1848373837064958, + 1.1652755181798344, + 1.4482605721386541, + 1.7090641904484354, + 1.9273940993343928, + 1.7914046912241552, + 1.9682234351900227, + 2.255052861817523, + 2.4759468246753933, + 2.422504559503542, + 2.5897139470282244, + 2.6980927396841867, + 2.6367787284783457, + 0.5624308402153666, + 0.6648993869614586, + 0.8453888350383831, + 1.2088065808156174, + 1.2146353329955089, + 1.384726431037592, + 1.6370377465936645, + 1.8442128222034295, + 1.9269222172231935, + 1.9895700195267167, + 2.1416914138162433, + 2.3436838976122853, + 2.3038333654587397, + 2.56347497061938, + 2.5839421846768467, + 2.7406937613823015, + 2.7335152395564215, + 2.7208558615082694, + 2.759292389502061, + 2.68217543030978, + 2.7179726667799553, + 2.7617280635395094, + 2.730951885751672, + 2.754462752982054, + 2.7499070609427996, + 2.777507181213892, + 2.7613928908240144, + 2.8018463731724257, + 2.6169926095850933, + 2.6648600868363808, + 2.7121821651151805, + 2.6195431147098063, + 2.7273334279259664, + 2.7664351682926505, + 2.752338477985635, + 2.7845386949907898, + 2.8242423333614557, + 2.5953757479705306, + 2.722341128757911, + 2.783459583865808, + 2.793119267582325, + 2.7611005412961784, + 2.6970158313017514, + 2.7795862055106912, + 2.8002762434063175, + 2.7940590064026782, + 2.754594980754449, + 2.7589887178106043, + 2.820215207756604, + 2.8037846111650584, + 2.814082283345453, + 2.607782244258445, + 2.7137422736740078, + 2.6518464842293525, + 2.6302864938361843, + 2.7800097648099538, + 2.789356328763075, + 2.790848397667771, + 2.7448102316498577, + 2.750949031316418, + 2.736862793871566, + 2.776203723843434, + 2.765810273332832, + 2.7843959652444994, + 2.783063789394399, + 2.716885256599962, + 2.751828984893981, + 2.4796443724023494, + 2.727129419516139, + 2.7075852726200242, + 2.7547280520942294, + 2.704958283011564, + 2.6691658996385996, + 2.783513680858956, + 2.7804805883023196, + 2.779080226274054, + 2.7713319718261396, + 2.679282894709134, + 2.7729853523662253, + 2.7865020032403507, + 2.817388363125321, + 2.5642347443046507, + 2.772532366469326, + 2.643083707117265, + 2.7029419125826912, + 2.4417103471266177, + 2.700028200313376, + 2.7314893865365697, + 2.7855347459373796, + 2.6231679460011, + 2.7673176283500176, + 2.811899850901378, + 2.7188362063043616, + 2.804209889765687, + 2.7981447644517345, + 2.7719238247088533, + 2.753661117101176, + 2.753412593909937, + 2.78686975268017, + 2.778988689805166, + 2.532137636047125, + 2.699271535310995, + 2.7702414646076132, + 2.634894554413151, + 2.7736206680115103, + 2.782864620910008, + 2.447531892592151, + 2.7997954738130666, + 2.651719548132826, + 2.659798462816701, + 2.7316429358979804, + 2.710417545163528 + ], + "dft_magnitude": [ + 318.395002210598, + 28.610130323450335, + 20.26512009268187, + 9.947079265177072, + 0.5020368765054722, + 9.254566480574312, + 12.759687374265475, + 12.254656715307872, + 13.309002995945852, + 9.588745744751161, + 6.530715736215407, + 2.6996378940460097, + 0.7362205014562527, + 2.821250174727885, + 5.60728571602385, + 6.414326695567279, + 5.649599974589197, + 6.172891604599396, + 4.109142242058516, + 2.4514190266470974, + 0.2727227998007537, + 0.8725270583420663, + 3.5405908817008354, + 3.819449580791532, + 3.626592217182337, + 3.385451029422578, + 1.465772385087135, + 2.1006034255821087, + 0.1360193498265616, + 2.744637966955684, + 2.1927769958425194, + 4.124004848249182, + 4.523412704229742, + 4.063489750134024, + 2.180334512505722, + 1.3743905260459661, + 1.1778639643641229, + 0.9995933285118528, + 1.867776206687283, + 2.103772434549697, + 2.5981555400722276, + 2.8702731725833512, + 1.994060712153237, + 1.00469799256983, + 1.0243532334008918, + 1.4196362978953445, + 2.5164281479327477, + 1.0271480446648023, + 1.549260849556277, + 0.9325121576556842, + 1.3608643244644292, + 1.0620324081765964, + 0.7527495374306786, + 0.519499219706248, + 1.1350800959918521, + 1.8923022858412974, + 2.1642420028423146, + 2.249343388055791, + 1.0256903489717974, + 0.7758615111944295, + 0.4209594934043468, + 0.5680057820384323, + 2.1789628990038628, + 3.374152910177925, + 1.9796136143936565 + ], + "mean_band_dot": [ + -5.41962299394072e-05, + -5.2691877499455586e-05, + -3.271623182854455e-05, + -1.931287596335096e-05, + -6.124106334937096e-06, + -1.2263619737495901e-05, + -1.9539584400263266e-05, + -2.8218157012815936e-05, + -1.793487345480571e-05, + -5.220358480073628e-05, + -3.013908872162574e-05, + -4.055221859289304e-05, + -3.389352673366375e-05, + -7.370729235844919e-07, + 3.921459665434668e-05, + 2.199084610765567e-05, + -4.079856012140226e-05, + -5.491199567586591e-05, + -2.9598914920825337e-05, + -3.94815428990114e-05, + -1.37377409146211e-05, + -9.143498687080864e-06, + -9.566847268160927e-06, + -1.9723313869235426e-05, + -4.235698895627138e-05, + -4.246056187184877e-05, + -5.9805426417369745e-05, + -3.462198958459339e-05, + -1.6750705071899574e-05, + 7.93704475654522e-06, + 9.973940223062527e-06, + 3.9667367445872515e-05, + 3.749013762899267e-05, + 4.076786103723862e-05, + 4.9577529352973215e-05, + 4.1205830484614125e-05, + 3.6556577924784506e-05, + 5.436048354567902e-05, + 4.5224811174193746e-05, + 4.396706162879127e-05, + 4.247402512191911e-05, + 5.102251077460096e-05, + 4.168561963524553e-05, + 5.027701558901754e-05, + 4.0098094586937805e-05, + 4.441126566234743e-05, + 3.704642995217e-05, + 2.3800345843483228e-05, + 4.465924803298549e-05, + 3.641860257630469e-05, + 4.527834516920848e-05, + 4.459452065930236e-05, + 5.939337893323682e-05, + 2.5787178628888796e-05, + 4.870867007866764e-05, + 4.7020409738252056e-05, + 5.564132743529626e-05, + 4.6279189291453804e-05, + 3.6147320997770294e-05, + 5.4993877029119176e-05, + 4.995910012439708e-05, + 4.599921567205456e-05, + 4.499922397371847e-05, + 6.147020121716196e-05, + 4.805566413779161e-05, + 6.636319517383527e-05, + 5.6748393035377376e-05, + 3.361876736107661e-05, + 3.705401536535646e-05, + 2.7632051569526084e-05, + 1.9157497263222467e-05, + 4.9042321734305006e-05, + 5.435089542515925e-05, + 5.628575490845833e-05, + 3.1130738079809817e-05, + 4.1625473386375234e-05, + 4.502179626797442e-05, + 5.152344846237611e-05, + 3.954605222133978e-05, + 4.8704226855988964e-05, + 5.261373485154763e-05, + 2.6276353992216173e-05, + 5.45515758858528e-05, + 4.704206958194845e-06, + 4.453928795555839e-05, + 2.590093049548159e-05, + 4.741039651889878e-05, + 3.2186034786718665e-05, + 2.8881712296424666e-05, + 5.775560111942468e-05, + 5.146373007391958e-05, + 4.251977134117624e-05, + 5.220760658630752e-05, + 3.7830660403415095e-05, + 4.8945869821181986e-05, + 4.281361270841444e-05, + 4.6228729843278416e-05, + 3.1388258321385365e-05, + 3.764800703720539e-05, + 3.757464742193406e-05, + 4.482354870560812e-05, + 1.711638196866261e-05, + 4.018696563434787e-05, + 5.521224602489383e-05, + 4.2925154502881924e-05, + 1.5508064279856626e-05, + 4.491687982977055e-05, + 4.982085533811187e-05, + 2.8229233521415154e-05, + 4.763465472024109e-05, + 5.7753391502046725e-05, + 4.131087052883231e-05, + 4.63402725472406e-05, + 2.933381301772897e-05, + 5.950330080395361e-05, + 4.7614777031412814e-05, + 2.7808659524453105e-05, + 4.3352647708161385e-05, + 5.3262957408151124e-05, + 3.4587592267598666e-05, + 5.061801493866369e-05, + 3.632731181824056e-05, + -6.076763838791521e-06, + 6.072940800549986e-05, + 2.5685770083327952e-05, + 3.087075765506597e-05, + 4.650034611586307e-05, + 4.802082435162447e-05 + ] + }, + { + "layer": 8, + "mean_snr": 0.9013515993716488, + "per_head_snr": [ + 0.8266009745792687, + 0.9180450054699206, + 0.9245231751860727, + 0.9302021137639067, + 0.9082801959286728, + 0.8734534489548437, + 0.8905951140081353, + 0.939112767082369 + ], + "mean_band_energy": [ + 0.7160174023260972, + 0.7168770439115053, + 0.9141606067757482, + 1.2807104378647098, + 1.433913807455835, + 1.6674678497008344, + 1.8483884771510022, + 2.063992811423839, + 2.0287780470286414, + 2.4127512167964014, + 2.4962753817662104, + 2.6478840582701473, + 2.3001181959078494, + 2.685526787393508, + 2.722868618535525, + 2.7463908573459825, + 0.6012571928361865, + 0.6586938229914625, + 0.8943414596611388, + 1.1350790634486105, + 1.411653808638027, + 1.660886996686477, + 1.8973904316582448, + 2.0152032978274477, + 1.9541061411018932, + 2.2355359512189406, + 2.363077686659662, + 2.7132199878974728, + 2.4361746075635633, + 2.709796118288006, + 2.745735742388643, + 2.6955739319518184, + 2.766378657851936, + 2.721091215963199, + 2.6853333609285723, + 2.720672562627639, + 2.7740943462270504, + 2.777641731154695, + 2.7708317512599177, + 2.76414587877842, + 2.7690507547953214, + 2.7597963225678157, + 2.768087791763957, + 2.7082767710658633, + 2.7359375241146537, + 2.7514586795669373, + 2.7688887135161533, + 2.758149786772343, + 2.7730175371492365, + 2.7790750383559573, + 2.759163210010069, + 2.753608239506983, + 2.7563224599514093, + 2.7874559492351665, + 2.7946963017790925, + 2.7897140991797436, + 2.7637559577829336, + 2.762233926868225, + 2.663140891361932, + 2.744170377403112, + 2.789978396090145, + 2.7732160107820665, + 2.6873485373650823, + 2.7767285281227263, + 2.7611773439603553, + 2.7875854174464383, + 2.6539254496037863, + 2.774428344837423, + 2.7571152903142124, + 2.7400076244785616, + 2.7677299074254087, + 2.7330492311056442, + 2.796400549975335, + 2.7537556031253896, + 2.7696506018600306, + 2.7643391896337786, + 2.7695910493568388, + 2.7659441973907217, + 2.7765434022906437, + 2.781615728987184, + 2.76683397376741, + 2.774740816268695, + 2.7826862094866627, + 2.7858152218936016, + 2.7529698330955767, + 2.6336212959985623, + 2.6212901591747197, + 2.665583270024527, + 2.7693701833663056, + 2.7651533344716697, + 2.758668330558592, + 2.7515932556874088, + 2.7633257959961437, + 2.674711545298482, + 2.783297395493973, + 2.776356393446946, + 2.7037836501501626, + 2.786736135915749, + 2.760378227055302, + 2.706520896699736, + 2.7177761144003103, + 2.792064241143507, + 2.787692235678465, + 2.689217990333118, + 2.648638546290888, + 2.788430775510287, + 2.7051219035535166, + 2.743877434316609, + 2.740260819270901, + 2.7556485894802165, + 2.7689341154636544, + 2.75985269061694, + 2.7740647613104716, + 2.7105653338089866, + 2.6641844941718436, + 2.7587980653118276, + 2.7519021469064224, + 2.776146937864956, + 2.7816564452806265, + 2.774031095106043, + 2.7626147322606904, + 2.7710687099456965, + 2.7576341791417, + 2.7077011249562393, + 2.6271402076762804, + 2.734212141607685, + 2.6877670552114807, + 2.7150807617129935 + ], + "dft_magnitude": [ + 324.55168765434416, + 25.00742528575836, + 18.245941740119385, + 9.295757407765137, + 0.5691838425775932, + 7.439579157607829, + 13.103744853057558, + 13.649424964609272, + 12.900601583713426, + 10.462062846494211, + 6.43460548752128, + 3.211448782476444, + 0.0703452124090771, + 2.275524315022807, + 4.115923036828215, + 6.480487662666337, + 6.435224360762216, + 5.624200939554312, + 3.8783259486638766, + 1.6460380621521387, + 0.22517415730767879, + 2.542277881299749, + 3.726075977533746, + 4.277407811321244, + 4.4590757401065435, + 3.643098336318359, + 2.638460909089053, + 1.4183274346141443, + 0.7713074460339743, + 1.2140197516625257, + 2.3878664000907874, + 3.832473161436072, + 3.5926487646939314, + 3.641833874738009, + 2.8002419673294647, + 1.6225985564391312, + 0.2504547457792899, + 1.3272038572294, + 0.797888747603468, + 1.3736446906290143, + 1.9349912057703054, + 2.0084596789954676, + 1.883582211569119, + 1.0969066050218903, + 0.23395039010155605, + 0.9660193348298988, + 1.367672383115051, + 1.7967900702176276, + 1.688532456058416, + 1.4527577386488326, + 1.4766706967129772, + 1.5069332801690718, + 0.1760776656287559, + 1.1318666773789394, + 2.2081602534492832, + 2.031097407706495, + 2.041008972077434, + 1.174197451296491, + 1.082670870590494, + 0.2911831366596784, + 0.24003054117680844, + 1.1975223192391982, + 2.101703264662063, + 2.9679233961860985, + 3.590869836443261 + ], + "mean_band_dot": [ + -6.658622623945121e-05, + -6.034613716110471e-05, + -3.897721137491317e-05, + -3.9915413026392343e-05, + -2.520367411307234e-05, + -2.7185073918190028e-05, + -2.5184419030210847e-05, + -4.026191243156063e-05, + -3.548156762178678e-05, + -4.632796412806783e-05, + -8.05755671251518e-06, + 1.2346470384727581e-05, + 1.486276900664052e-06, + 3.2370555175020854e-05, + 4.613187294921772e-05, + 6.440972745735962e-05, + -5.454887229916494e-05, + -5.817111264150299e-05, + -4.53415680112812e-05, + -3.5495390932283044e-05, + -2.719192031008788e-05, + -3.154477261091415e-05, + -2.4983253638310998e-05, + -4.0539844121667556e-05, + -4.103246067188593e-05, + -2.5643503704486648e-05, + -2.738478890762508e-05, + -1.2212281944812275e-05, + 2.636385630694349e-06, + 3.571520889522617e-05, + 4.7383409395251874e-05, + 4.4929500347734574e-05, + 5.9104257957187656e-05, + 6.351834815632174e-05, + 5.142191322704548e-05, + 4.854304580703683e-05, + 6.053759466340125e-05, + 6.235176397240139e-05, + 6.13795053254762e-05, + 6.369442621689814e-05, + 5.395967244226085e-05, + 5.1359165922804095e-05, + 7.165464671743393e-05, + 4.1738481343145395e-05, + 5.176751062663243e-05, + 6.911802393005928e-05, + 5.326132747995871e-05, + 6.384232051459549e-05, + 6.357690983804787e-05, + 5.7295578699267935e-05, + 5.706982477704514e-05, + 6.939976242392731e-05, + 6.0714651567650435e-05, + 6.285881204348698e-05, + 6.061008039637272e-05, + 7.299132826688037e-05, + 5.8419883714577736e-05, + 5.5047191153789754e-05, + 5.214890620663937e-05, + 5.723002436752722e-05, + 5.7911792445963783e-05, + 5.540713854657042e-05, + 5.0687232828749984e-05, + 6.513341293157282e-05, + 6.148242195536113e-05, + 6.035656565472891e-05, + 3.861656523440615e-05, + 6.165231945942651e-05, + 5.828934337159808e-05, + 6.660623364496132e-05, + 5.1177234851707e-05, + 7.0417839538095e-05, + 5.377506886361516e-05, + 6.481249079115514e-05, + 5.8867860616373946e-05, + 7.329878542350343e-05, + 6.468782930824091e-05, + 5.4086668853869924e-05, + 7.023295165708987e-05, + 5.9552738889578904e-05, + 5.6877167935454054e-05, + 5.704651107407699e-05, + 6.483089623543492e-05, + 5.877268404219649e-05, + 5.323910977494961e-05, + 4.008663387367051e-05, + 5.1816616945643545e-05, + 4.2858249628352496e-05, + 5.467186562668758e-05, + 5.896661343740561e-05, + 5.736883309737095e-05, + 5.260860984890314e-05, + 4.184650532579326e-05, + 4.953650594075043e-05, + 6.528208169243044e-05, + 5.9680969741293666e-05, + 5.204249771395553e-05, + 7.8964131603243e-05, + 5.7864823247655295e-05, + 6.187376024513469e-05, + 4.9092106564785354e-05, + 5.206837599303071e-05, + 6.734717362633091e-05, + 6.531363544581836e-05, + 4.338489600286266e-05, + 7.082287157800238e-05, + 5.129174664375569e-05, + 5.9066881362923596e-05, + 7.310463854537375e-05, + 6.459554364823816e-05, + 5.985629508131751e-05, + 5.7702535855241877e-05, + 6.533700519639751e-05, + 5.70500632193216e-05, + 4.9378877349681716e-05, + 5.38170969548446e-05, + 6.072080077501596e-05, + 7.204806559002463e-05, + 5.4655646835044536e-05, + 5.8410857263879734e-05, + 5.137426717283233e-05, + 6.072896212572232e-05, + 5.8628306305763545e-05, + 5.564255354784109e-05, + 3.79563820729345e-05, + 6.533596967983613e-05, + 6.173353267513448e-05, + 5.313128281159152e-05 + ] + }, + { + "layer": 9, + "mean_snr": 0.910651115315432, + "per_head_snr": [ + 0.8620220015643318, + 0.8989638430472722, + 0.9578474357579625, + 0.9263663430919225, + 0.8691968403756719, + 0.8743658112813626, + 0.9626081219223692, + 0.9338385254825633 + ], + "mean_band_energy": [ + 0.6346489378810962, + 0.6615689522954931, + 0.9401230749310443, + 1.198161173929349, + 1.467845492246898, + 1.5969565293462384, + 1.8870791031009766, + 1.9437949028716908, + 2.0434587855809454, + 2.1339577511778014, + 2.5193551688480493, + 2.3127878954768093, + 2.3008536557772192, + 2.484798031347337, + 2.454603156140921, + 2.451269117381049, + 0.651932799550667, + 0.7073800999170983, + 0.9044902749045662, + 1.2836058016708112, + 1.362408161014426, + 1.6970989229336961, + 1.7759660993145552, + 2.101792905341762, + 2.07281739143421, + 2.0742957982636536, + 2.4756723696698533, + 2.3128984292390182, + 2.293027715087282, + 2.4998194119820383, + 2.4591894532147602, + 2.480572952389833, + 2.4349488274145674, + 2.459668099822382, + 2.4552712798227496, + 2.481267632348139, + 2.4729356401157894, + 2.4504996567184882, + 2.3115454341188784, + 2.471522199921635, + 2.4631707953701802, + 2.354759652288392, + 2.4899817099634003, + 2.4681219232811458, + 2.4291166471458396, + 2.403523927777158, + 2.4907220677634605, + 2.4422588592929237, + 2.4751867557228877, + 2.4698685728973295, + 2.4552991694608455, + 2.477985665730957, + 2.510426957285679, + 2.4702840620852218, + 2.4474591704827233, + 2.44809275553429, + 2.4552390206581176, + 2.438495206349778, + 2.434551646728389, + 2.459774686436239, + 2.48480461762251, + 2.4802586519705, + 2.386271225902899, + 2.4791562845785613, + 2.440999629731918, + 2.4269216539936505, + 2.354667311368102, + 2.4238278430144202, + 2.427703450902106, + 2.4493608491005876, + 2.499069556814348, + 2.450558495524568, + 2.4226725439449996, + 2.4776469904468037, + 2.460922061695893, + 2.4835300718517352, + 2.3711448403891495, + 2.4685779866552746, + 2.4427551966039136, + 2.473822506570036, + 2.4613142411383304, + 2.4540161540387473, + 2.4753154566094793, + 2.4525191334689787, + 2.375346603284843, + 2.3680435697932722, + 2.4881558449624706, + 2.4213182343031665, + 2.4114290310240154, + 2.4628669768437472, + 2.4535300460473746, + 2.453345908367389, + 2.505830468658923, + 2.4535029518556115, + 2.453394371286466, + 2.396672881885335, + 2.415357414825193, + 2.458625773371173, + 2.4680157862395173, + 2.47497099646516, + 2.4648871716870033, + 2.439287694267552, + 2.4907498829391237, + 2.4886486704740065, + 2.481728401415216, + 2.4905392792867698, + 2.4332996661666524, + 2.44933591008094, + 2.464308959140631, + 2.4864230968997507, + 2.3901437841131976, + 2.4310327642550362, + 2.452138216058877, + 2.4059002680092725, + 2.420417543450636, + 2.4633658130220706, + 2.4217289853988806, + 2.5008154514762655, + 2.3571550972765767, + 2.4617476531698745, + 2.402601613982311, + 2.4738921388518484, + 2.438942700706539, + 2.478373299553681, + 2.4481121876929546, + 2.4798526824428357, + 2.4733154465327356, + 2.414090128902254 + ], + "dft_magnitude": [ + 293.2172864572034, + 18.471833980230766, + 13.941536224750688, + 6.73904410943395, + 0.07574322499809866, + 5.52568693612603, + 10.666318867536917, + 12.191960022918657, + 11.80923110150632, + 9.191669716354845, + 6.553385446657775, + 3.498576179229195, + 0.4111939249969618, + 2.7344412247909, + 4.821369087920348, + 5.576330328728372, + 5.5701479344765135, + 4.762727935797924, + 3.3667570089944934, + 1.536709672718108, + 0.4112132754801169, + 1.5193862104811944, + 2.592009672906868, + 4.11621664725426, + 3.756600497427412, + 3.189313878311807, + 2.4350936351822825, + 1.1047901392352186, + 0.2715247761708273, + 1.2321267972714993, + 2.248840267930103, + 3.1751575998352632, + 3.4078525177350185, + 2.6547713059086475, + 2.6219377144805387, + 1.0848281247533924, + 0.15539947954530223, + 1.2526895201965327, + 1.4688206814344036, + 0.7357742833442551, + 1.0674516597336017, + 2.1881120933635807, + 1.2182067414595699, + 0.5699993616465379, + 0.2245960118213491, + 1.1814446143164374, + 1.6378501287605702, + 2.5689080020602764, + 2.0089752193010977, + 1.400604422199065, + 0.8442708989389766, + 0.7493096097947985, + 0.49762098673115457, + 0.6393873557585452, + 1.289904948249313, + 1.3673392988206943, + 2.0911394001660586, + 2.1206029181208814, + 1.9954898881190324, + 1.1723170686237239, + 0.5551871832625711, + 0.5293636275587413, + 1.194412020759529, + 1.539898080277825, + 2.202174224473879 + ], + "mean_band_dot": [ + -4.555525197247334e-05, + -3.596086583002034e-05, + -1.845603782157923e-05, + -2.7683999405780924e-05, + -2.0735722344511487e-05, + -9.718565110006239e-06, + -2.672360108135763e-05, + -3.694824067679292e-05, + -4.2254353502357844e-05, + -5.715333372791065e-05, + -2.148257647149876e-05, + 7.893510655776481e-06, + 2.0946081406236772e-05, + 4.1760662952583516e-05, + 4.5337772405673604e-05, + 4.3630609582123725e-05, + -4.007010556961177e-05, + -4.714903589331243e-05, + -2.3180418907031708e-05, + -2.7982541297433272e-05, + -8.830962798356268e-06, + -2.0189151911154113e-05, + -3.098322679306875e-05, + -4.220923142383981e-05, + -3.168988317270305e-05, + -4.1273136503150454e-05, + -8.107887879305054e-06, + 2.6186113473158912e-05, + 4.481189705529687e-06, + 3.278322981259407e-05, + 4.202525961716219e-05, + 4.081924379306656e-05, + 4.326301359469653e-05, + 3.929751144937654e-05, + 4.835199445096805e-05, + 4.295041418345136e-05, + 4.638502866782801e-05, + 4.374629787662343e-05, + 3.220939925085986e-05, + 3.508981262712041e-05, + 4.5642145209967566e-05, + 3.0319030088321597e-05, + 4.837844130634039e-05, + 5.0128454915920884e-05, + 3.931721892058704e-05, + 3.8565319300687406e-05, + 4.63419789866748e-05, + 4.1693971866152424e-05, + 4.7221430861554836e-05, + 4.6537098114640685e-05, + 4.77003109153884e-05, + 4.3000270920856565e-05, + 4.773302669036639e-05, + 4.368909344520944e-05, + 4.1268735003541224e-05, + 4.138171834711102e-05, + 4.726373337859968e-05, + 3.900359183717228e-05, + 4.54334849564475e-05, + 4.464565972739365e-05, + 4.960504034556834e-05, + 5.010083214074257e-05, + 3.7663196621906536e-05, + 5.153988953310318e-05, + 4.728506860374182e-05, + 4.164686492913461e-05, + 3.656183986322503e-05, + 3.881574934894161e-05, + 4.304907895402721e-05, + 5.086985288471624e-05, + 4.5291199057828635e-05, + 4.6014312033548777e-05, + 4.640109716547158e-05, + 5.124292692926247e-05, + 4.7526534899589024e-05, + 4.345402430772083e-05, + 3.087852238081723e-05, + 3.979529105890833e-05, + 5.090012996333826e-05, + 4.7289393933169777e-05, + 4.716914008895401e-05, + 4.5998656105439295e-05, + 4.8734596248323214e-05, + 4.12666885267754e-05, + 3.554856459686562e-05, + 2.650115919777818e-05, + 5.2375144861116496e-05, + 4.450917515441688e-05, + 3.204593733130423e-05, + 5.286751138555701e-05, + 4.2881503077296657e-05, + 4.4298832790445886e-05, + 5.26952768495903e-05, + 4.5612940994033124e-05, + 3.9423072792033054e-05, + 3.883112546532175e-05, + 4.0948161199594324e-05, + 4.49611475232814e-05, + 4.754025270869988e-05, + 5.408231754699955e-05, + 4.8207290888058196e-05, + 3.6806095408792316e-05, + 5.677012381966051e-05, + 4.4549692063355906e-05, + 4.394579775635066e-05, + 4.956715203263684e-05, + 3.379005113401945e-05, + 4.071216812917555e-05, + 4.884427630713617e-05, + 5.299666429436911e-05, + 4.216043078031362e-05, + 3.7486207418169215e-05, + 5.3231456774938124e-05, + 4.403135267239122e-05, + 3.572411264940456e-05, + 5.236693286292393e-05, + 4.4545121511418984e-05, + 4.3086810563863764e-05, + 3.8256190634911036e-05, + 5.014801746483499e-05, + 3.54443185983655e-05, + 5.0721313129997725e-05, + 3.989939992266045e-05, + 5.0392317007208476e-05, + 4.1714068686360406e-05, + 4.378919470582332e-05, + 4.444418988214238e-05, + 3.9991143609086066e-05 + ] + }, + { + "layer": 10, + "mean_snr": 0.8991914311462631, + "per_head_snr": [ + 0.8876662174593188, + 0.9118227051833573, + 0.9531930518678238, + 0.9513098648649763, + 0.9144567993031824, + 0.8563851074916723, + 0.822828038877974, + 0.8958696641218004 + ], + "mean_band_energy": [ + 0.7308032905987836, + 0.8528260498083222, + 0.982004260505712, + 1.3247141536185811, + 1.3310974475603512, + 1.5849818035780396, + 1.7550626964312421, + 2.0662704281098065, + 2.0118668674953426, + 2.041919756741107, + 2.290754627207779, + 2.3564759619259785, + 2.5257671484039346, + 2.506191716571811, + 2.589333599547361, + 2.5702062097835157, + 0.8473205312786503, + 0.7919748637689126, + 1.0662076475877234, + 1.19910209680488, + 1.2886305670719427, + 1.5150758687300057, + 1.75782368078616, + 1.9375167581508723, + 1.9777446497275548, + 2.0661567798811387, + 2.353967496117912, + 2.325656742910506, + 2.4663643910400275, + 2.531942417028887, + 2.466535118310409, + 2.4850996555361426, + 2.6100593521281414, + 2.513730166142187, + 2.631705844125273, + 2.6361686775631306, + 2.635257096248509, + 2.628207316909384, + 2.614819354041348, + 2.532208034426562, + 2.620515680577201, + 2.620129686739176, + 2.5092762612291875, + 2.6344943430629204, + 2.5400521658820097, + 2.551240843635254, + 2.6132574337534464, + 2.6205340196458895, + 2.628621506389271, + 2.572478834207354, + 2.611595222544997, + 2.616731158891506, + 2.63015266917815, + 2.6455200068916334, + 2.538587307442805, + 2.6012947270660742, + 2.625727926812843, + 2.6285047604538097, + 2.553182402895221, + 2.6326594237517273, + 2.5446183658166444, + 2.586394749670049, + 2.5715536802638557, + 2.5656932585249432, + 2.626749485241099, + 2.6362433522682487, + 2.596543609561314, + 2.6149600516365386, + 2.6033014589849532, + 2.6210954935069015, + 2.5907327691808995, + 2.6488713810821043, + 2.6146349946872327, + 2.558469188069477, + 2.5998169638466084, + 2.646653203533937, + 2.636317637711313, + 2.6382682039605605, + 2.6390097157187657, + 2.4804381393743764, + 2.6077667819060544, + 2.5604614501918945, + 2.6142584548177634, + 2.6649364953511014, + 2.6283591853119734, + 2.6259537323447244, + 2.6001169994878417, + 2.621670140930389, + 2.598399115606167, + 2.503689710697415, + 2.612605295654637, + 2.632632443549886, + 2.5877974116063562, + 2.626678375013883, + 2.6274633546322708, + 2.6501195811076155, + 2.6211177361942735, + 2.621551251313562, + 2.53947521552484, + 2.635212120443736, + 2.6378564887905114, + 2.6428597468811823, + 2.6418870956531944, + 2.602193254939511, + 2.5683633467660805, + 2.6228044695320953, + 2.6244365372209106, + 2.6515543216516733, + 2.517006219800228, + 2.635446854065025, + 2.4766394960824787, + 2.6242690008465743, + 2.5402549933500254, + 2.627904044993871, + 2.6074754918619654, + 2.6227630594468767, + 2.62362346427712, + 2.6400087274451494, + 2.65755290089065, + 2.623176468169806, + 2.626216488290064, + 2.6213474894326705, + 2.6483504917074665, + 2.5619118046773526, + 2.6389813163725186, + 2.6304672604420336, + 2.6345680426595575, + 2.595205054577315 + ], + "dft_magnitude": [ + 308.7398619904045, + 22.459494264109026, + 16.257843265319394, + 8.19326532597575, + 0.17328942802922048, + 6.763432920609124, + 11.611602321777548, + 12.6399565129809, + 10.935968263981989, + 9.021603291610901, + 5.976669626530874, + 2.6577866208225878, + 0.22527303436165685, + 3.283245601906601, + 4.2794325748288395, + 5.4710119374574955, + 5.719705453220013, + 4.650709204097203, + 3.7784566831066297, + 1.650374213824784, + 0.18940996814574648, + 2.129067613034126, + 2.147889232401203, + 2.9845757838929226, + 3.1171649786221827, + 2.9569276262118263, + 2.3549193177211394, + 1.1496563677239193, + 0.10371568505088559, + 1.4528092810118864, + 2.4605438358379845, + 3.213471628985706, + 3.1799196532142155, + 2.7620553331551694, + 2.097322763391592, + 1.3358925238859585, + 0.361455892209123, + 1.1275883886191422, + 1.4265887807828705, + 1.9986722944418798, + 1.662338192349653, + 1.7661852312271074, + 1.267345355004901, + 1.087863766628287, + 0.2623943793058163, + 1.0406931018355523, + 1.88484379848822, + 2.4183611592499474, + 1.5004826882481277, + 2.0802686254850644, + 1.1069214612617757, + 0.43835539761126935, + 0.42543922670046197, + 0.7595496525476004, + 0.8691489032313717, + 2.007864757265578, + 1.6604500439945151, + 1.984001899376618, + 1.4402407412550562, + 0.3857625279007635, + 0.4906746245893637, + 1.032237050152021, + 1.3242574732403745, + 1.6340855521580584, + 2.1239722936106773 + ], + "mean_band_dot": [ + -1.7078895893973822e-05, + -1.7184033936246124e-05, + 1.6411834167229244e-06, + -1.1661074267976801e-05, + 3.025314754268038e-06, + -1.4229181118707857e-05, + -7.321586508624023e-06, + -1.645029476549098e-05, + -8.47844216878002e-06, + -2.535961289140687e-05, + -1.530854342490784e-05, + 2.220805868091702e-05, + 3.412524404211581e-05, + 6.0425141327868914e-05, + 9.573235183779616e-05, + 0.00010365460821049055, + -1.1350618592587125e-05, + -1.0197130791311793e-05, + 2.0290411839596345e-06, + -6.948682596430444e-06, + -9.213040357280988e-06, + -8.477060589484608e-06, + -3.3817788107626257e-06, + -2.3852912590882624e-05, + -4.4875559979118407e-07, + -5.749358820139605e-07, + 6.551888873218559e-06, + 1.4091490129430895e-05, + 3.6313366990725626e-05, + 6.948820009711199e-05, + 8.821805977277108e-05, + 7.689777430641698e-05, + 9.922991375788115e-05, + 9.265655444323784e-05, + 0.00010113160192304349, + 9.44883176998701e-05, + 0.00010961334555759095, + 0.00010521000103835831, + 0.00010477880050530075, + 9.005215315482928e-05, + 0.00010002186689916925, + 0.00010669957373465877, + 9.242760665983951e-05, + 0.0001016639207591652, + 8.883596183295595e-05, + 0.00010470653160155052, + 0.00010183066569879884, + 0.00010508875743653334, + 9.900493290615486e-05, + 9.658656517785857e-05, + 8.990636194994295e-05, + 0.00010393678758191527, + 9.968720769393258e-05, + 0.00010838078856068023, + 9.36137525968661e-05, + 9.418695734098037e-05, + 9.386196461491636e-05, + 0.00010785734866658458, + 9.406009826307127e-05, + 0.0001035808395499771, + 8.639190536996466e-05, + 0.0001112930790441169, + 8.186835543710913e-05, + 9.887264491226233e-05, + 0.00010430621205159696, + 0.00010664600449672434, + 0.0001023876243380073, + 9.905563342726964e-05, + 0.00010430767997604562, + 9.888471004160237e-05, + 9.618120611776249e-05, + 9.595278470442281e-05, + 0.00010478609874553513, + 0.00010127003304205573, + 0.000100075908449071, + 0.00010387265979261429, + 9.650395162452696e-05, + 0.00010238524419037276, + 0.00010294229696228285, + 9.841180553848972e-05, + 0.00010621095862006769, + 8.584925990362535e-05, + 0.00010742990889411885, + 0.00010193708658334799, + 0.00010377441128639475, + 0.00010029310169556993, + 9.55325558891218e-05, + 0.00010061635202873731, + 9.852122161646548e-05, + 9.083652457775315e-05, + 0.00010521359013182519, + 9.571474197400676e-05, + 0.00010576825866337458, + 0.00010041967016150011, + 0.0001031888627949229, + 0.00010158961185879889, + 0.00010486577070878411, + 0.00010882722608585027, + 9.208376059177681e-05, + 0.00010869460652429552, + 0.00010466554476806778, + 0.00010079166895593517, + 0.00010184849270444829, + 0.00010193138814429403, + 8.319644030052586e-05, + 0.00010294263847754337, + 0.00010627133860907634, + 0.00010922018782366649, + 0.00010160867304875865, + 0.00010899890457949368, + 7.776265886150213e-05, + 9.685616123533691e-05, + 8.367347697912919e-05, + 0.00010354456253480748, + 0.00010318946624465752, + 0.00010482474544915021, + 0.00010388375426373386, + 0.00010468994355505856, + 9.923864263328142e-05, + 0.00010655256915015343, + 0.000110556107983939, + 0.00010396338620921597, + 0.0001040490124637472, + 9.736002084537176e-05, + 0.00010218257466476643, + 9.925857193593401e-05, + 0.00010023631625699636, + 9.72138645920495e-05 + ] + }, + { + "layer": 11, + "mean_snr": 0.8875157408513284, + "per_head_snr": [ + 0.8963154519143034, + 0.8933818599666824, + 0.8965628282409275, + 0.9269467634229989, + 0.9294389940103339, + 0.864935131854869, + 0.8604012168981104, + 0.8321436805024021 + ], + "mean_band_energy": [ + 0.6604969228871749, + 0.7340977399199295, + 0.833505360530193, + 1.01644555537183, + 1.184185429980682, + 1.3320386085101399, + 1.4496137561780227, + 1.61116776562411, + 1.63812438081019, + 1.8199381072903602, + 2.231976859208098, + 2.3980441714978937, + 2.448824135159823, + 2.6221050690081213, + 2.529222688276791, + 2.5105334125485665, + 0.7207550139379646, + 0.6627792938455808, + 0.8146451947399442, + 1.0121575389571706, + 1.2152639092561817, + 1.3168428417127096, + 1.559176035618803, + 1.6270655868590227, + 1.6963613380444347, + 1.9178753310677203, + 2.329943108433186, + 2.547647077605216, + 2.6502681555140466, + 2.473007480516518, + 2.5194932969062176, + 2.6179365472953364, + 2.588396369289157, + 2.6361488342109602, + 2.604632253697325, + 2.5818710797610347, + 2.5623897457135545, + 2.5845780216584178, + 2.5486294427456766, + 2.5734320028632958, + 2.617316736360232, + 2.620867738028986, + 2.537293189732777, + 2.6165353946048153, + 2.596927296539011, + 2.5947365283423522, + 2.55782837015116, + 2.5566578895503156, + 2.6050229325745917, + 2.5720564671063126, + 2.5833547821370964, + 2.5836900831349467, + 2.5032132658281547, + 2.586342093655041, + 2.6181074491710827, + 2.5805370504089655, + 2.592654217295907, + 2.5924010923698564, + 2.6005760905639796, + 2.608850549551141, + 2.5622395690826067, + 2.593606144172501, + 2.5880249390536, + 2.5957501294870564, + 2.604087257327379, + 2.579892436541467, + 2.5801670995599526, + 2.597712032856329, + 2.5882436541617313, + 2.5880322322567473, + 2.5822444450777553, + 2.584992190709089, + 2.617391068912408, + 2.6129487881685645, + 2.586979119477334, + 2.5872192343907088, + 2.594980413255916, + 2.5776282153008836, + 2.5933442243150466, + 2.5948724857633643, + 2.603661236300022, + 2.585839193498421, + 2.601704224266209, + 2.579084473238426, + 2.5890146464964197, + 2.6154277464027373, + 2.5821722548008665, + 2.642298562296739, + 2.590279744223226, + 2.545753500292042, + 2.5955227863190196, + 2.581716538332145, + 2.613152564642478, + 2.623779141214701, + 2.579604937670812, + 2.543077050649062, + 2.5889436315089522, + 2.5662706546639367, + 2.5713999210228433, + 2.5012743899678025, + 2.6233553152305564, + 2.5699498750862286, + 2.5595261542627252, + 2.5788964712853044, + 2.6101382699476865, + 2.563250968938073, + 2.583364274252135, + 2.6040480134981614, + 2.559892049808594, + 2.5915616150285548, + 2.573975299531666, + 2.572795499032974, + 2.561565263230216, + 2.641425030439706, + 2.510791017081109, + 2.5953026047812457, + 2.5733092345647783, + 2.565725329599042, + 2.5788564995540444, + 2.598834254015614, + 2.60104263624938, + 2.5525207044411733, + 2.6024885074826347, + 2.6077212758888635, + 2.6251905147699475, + 2.5651964457123757, + 2.429258968436116, + 2.573999157715951 + ], + "dft_magnitude": [ + 302.7908988077003, + 25.942263720585235, + 18.876646918809673, + 10.305754428751744, + 0.6049447641708504, + 7.799255435033257, + 12.6543263693715, + 14.476825964950221, + 13.384936704182522, + 9.68655352281167, + 5.675545514874207, + 2.47587453115469, + 0.3628019230230491, + 2.554078243628799, + 3.9083663793127745, + 5.130909220866576, + 5.662471519400809, + 5.516381345205042, + 4.3921249393876955, + 2.4573885049097837, + 0.5484792643320586, + 1.9284151514906034, + 3.0668830145126207, + 3.0656106961003475, + 2.3793853746779754, + 2.246976137750843, + 2.0298727963305234, + 0.9539214495278971, + 0.6728829575692247, + 1.0170476602739538, + 2.4538454495452005, + 2.9027943538139827, + 2.8830554049077732, + 2.752245201736373, + 2.0557252506864843, + 1.1394174201498521, + 0.11683073523540814, + 0.6357488710202923, + 1.130271890926186, + 2.0711703166670308, + 1.899989670089703, + 1.2739442648911126, + 1.2747923887741583, + 0.801641586612864, + 0.6292796926656231, + 0.9696539140156111, + 1.5744571386695068, + 2.0022178157016137, + 2.0346248026903306, + 2.184215935059775, + 1.4069926409899118, + 0.6430771243015093, + 0.4171692705893067, + 0.8134039283461305, + 1.4659168173940635, + 2.0744898801720875, + 2.1617668627986815, + 2.096790259712704, + 1.3249019649914682, + 0.3431979219597931, + 0.22463336494034508, + 0.8604292825906662, + 1.4781089710988915, + 1.8239464510824173, + 1.9826798693850378 + ], + "mean_band_dot": [ + 4.604393666340911e-06, + -3.89242479315044e-06, + 9.564046649757074e-06, + 1.3356209507264794e-06, + 5.161693479749374e-06, + 8.74422994456836e-07, + -1.360315297915804e-05, + -2.1063110011709796e-05, + -1.7167566397802148e-05, + -1.1759604262806533e-05, + -2.7774573652550316e-05, + 2.877861885508537e-05, + 1.7867451902020548e-05, + 8.360729316336801e-05, + 8.08928799642672e-05, + 8.605252287452458e-05, + 1.3175670687815e-06, + 9.625578712757488e-06, + 7.726671583441203e-06, + 2.459942152199801e-07, + 1.2841384773309983e-06, + -9.563787983779548e-07, + -1.349581236809172e-05, + -2.2935964835824052e-05, + -2.4755902586548473e-05, + -1.591807813383639e-05, + -3.209936210168962e-05, + 1.528750819090874e-05, + 5.071979899184953e-05, + 5.2603952781282715e-05, + 8.370307250515907e-05, + 0.00010341690858695074, + 9.369613781018415e-05, + 9.954694382940943e-05, + 0.00011315450319671072, + 0.00010245004523312673, + 0.00010492122737559839, + 0.00010160254259972135, + 9.013798012347252e-05, + 9.518206888969871e-05, + 0.00010561693397903582, + 9.993353069148725e-05, + 9.624289816656528e-05, + 9.965215167540009e-05, + 0.00010204262707702583, + 0.00010332924830436241, + 0.00010214275835096487, + 0.00010461742567713372, + 0.00010052663446913357, + 8.759398042457178e-05, + 9.105667277253815e-05, + 9.503294131718576e-05, + 9.444505622013821e-05, + 9.799916642805329e-05, + 9.163824188362923e-05, + 0.00010258574593535741, + 0.00010322487378289225, + 0.00010089388615597272, + 9.972914449463133e-05, + 0.00010197010578849586, + 8.493280029142625e-05, + 9.595441269993898e-05, + 0.00010672975236047932, + 0.00011068676212744322, + 0.0001021847119773156, + 9.999720623454778e-05, + 9.194599897455191e-05, + 9.741624216985656e-05, + 9.187205569105572e-05, + 0.00010279958360115415, + 0.00010578655883364263, + 0.00010512198014112073, + 0.00010595137882773997, + 0.00011254544551775325, + 0.00010673112319636857, + 0.00010108661081176251, + 0.0001007209457384306, + 0.00010282731318511651, + 0.00010901848690991756, + 9.211435963152326e-05, + 0.0001049963843797741, + 9.209267091137008e-05, + 9.507260824648256e-05, + 9.466581695960485e-05, + 0.00010511780601518694, + 0.00010484755011930247, + 9.999406984206871e-05, + 0.00011125253695354331, + 0.00010947079590550857, + 8.312531724641303e-05, + 0.00011063461261073826, + 9.762884974406916e-05, + 0.00010351468836233835, + 9.589585079083918e-05, + 0.00010816401481861249, + 8.811038742351229e-05, + 0.00010411479843241978, + 9.534900254948298e-05, + 0.00010826837205968332, + 8.959577803580032e-05, + 9.919494141286123e-05, + 0.0001044247096615436, + 9.139256371781812e-05, + 0.00010012882557930425, + 0.00011202511723240605, + 9.681785286375089e-05, + 0.00010017109207183239, + 9.995190384870511e-05, + 9.238043594450573e-05, + 9.127941984843346e-05, + 9.412971917299728e-05, + 9.951403853847296e-05, + 9.949207196768839e-05, + 0.00010471951054569217, + 9.014645479510364e-05, + 0.00010711957793319016, + 8.012631951714866e-05, + 0.00010363053524997667, + 0.0001004334922072303, + 0.00010736139256550814, + 9.306389029006823e-05, + 9.939026949723484e-05, + 0.00010054684298665961, + 9.733370870890212e-05, + 0.0001127107570937369, + 9.090210096474038e-05, + 7.178697478593676e-05, + 9.609169137547724e-05 + ] + }, + { + "layer": 12, + "mean_snr": 0.8989559341273003, + "per_head_snr": [ + 0.87396766364343, + 0.9460638086286272, + 0.9186538258121193, + 0.8710382835329654, + 0.9083605399656103, + 0.948717270252311, + 0.8642092763785755, + 0.8606368048047645 + ], + "mean_band_energy": [ + 0.7454368463588703, + 0.7951320375190853, + 1.0105059769234295, + 1.2547312029472373, + 1.4507852510670238, + 1.5712126240610447, + 1.8985257042829549, + 1.8964878506977332, + 1.935567466100128, + 2.112147524022083, + 2.465147544726065, + 2.285572761436555, + 2.515127280838338, + 2.6598208304516966, + 2.682785111262536, + 2.5810566376636412, + 0.8037558644362512, + 0.7324602651647165, + 0.9436247765972521, + 1.2959836083773708, + 1.3724071111695562, + 1.6296026082425215, + 1.8085555033832348, + 1.9758624732785846, + 2.0017271274267916, + 1.8659120404588876, + 2.4775973678234795, + 2.2878083278553802, + 2.595962777117064, + 2.6417687311218945, + 2.647208854316931, + 2.6944813260692726, + 2.574559609132857, + 2.6388018692300257, + 2.710601711554462, + 2.6887030397906067, + 2.704579693348947, + 2.742920130830223, + 2.69530936719803, + 2.6590883523420192, + 2.6836386924677402, + 2.7199219672944235, + 2.7296320540290253, + 2.673278924756689, + 2.7128358996446575, + 2.6056060939669496, + 2.6870573839920375, + 2.6906377365027794, + 2.701420746858698, + 2.710196906673689, + 2.6769525988636786, + 2.69355357020968, + 2.66622957761372, + 2.5294134255686966, + 2.6629942209014867, + 2.681809989378989, + 2.7069579920281033, + 2.685824304382665, + 2.713801701917891, + 2.703935132538721, + 2.666074508162694, + 2.6314074375220695, + 2.7049248527341163, + 2.665090916715293, + 2.697890515927879, + 2.72774895366865, + 2.6124753879975646, + 2.688373394133567, + 2.6663696307155185, + 2.7223810479731316, + 2.7109566179697087, + 2.709630861584797, + 2.710732838623974, + 2.7142349204216156, + 2.6881671095173605, + 2.6698310295281615, + 2.6715073137824614, + 2.6970822876958245, + 2.7431213802742516, + 2.729295588522181, + 2.720244356571195, + 2.7079759741512692, + 2.7124248133178437, + 2.719692078388203, + 2.717297115757944, + 2.661116283160215, + 2.7021628133551943, + 2.6987331416498037, + 2.7128587454852173, + 2.7361019128145205, + 2.6928601739306455, + 2.644796975122496, + 2.6811976702168447, + 2.7019578803196254, + 2.5967082759585836, + 2.6588001877869853, + 2.584836562076303, + 2.689794753903419, + 2.609203382721933, + 2.725028546997299, + 2.6811418015747055, + 2.698294562600255, + 2.686529896101632, + 2.711361989712101, + 2.7089815350145443, + 2.633470889388869, + 2.661713307010933, + 2.5629415309727133, + 2.686816336621524, + 2.7225021168356918, + 2.6610589793103294, + 2.537815210625503, + 2.703590415810069, + 2.709749418608043, + 2.647900804964408, + 2.7030013213162665, + 2.6975471005012324, + 2.6973509135700535, + 2.6992232772573637, + 2.5974387194564965, + 2.6268701429162835, + 2.694170011804543, + 2.7067080875483125, + 2.71051578016049, + 2.6989106466254302, + 2.734183447332869, + 2.7265239658863134, + 2.695487940065883 + ], + "dft_magnitude": [ + 317.1879124929643, + 24.1384771568125, + 16.929849697082233, + 9.537502315112084, + 0.2632954426179922, + 7.204078345734904, + 11.338750830210994, + 13.045624734497327, + 11.668763179254997, + 8.902105065226792, + 6.434999479837744, + 3.388255916879553, + 0.7957976079026379, + 2.255181942304585, + 4.790889476838322, + 6.623480936156099, + 6.399028095393709, + 5.862922072557905, + 3.993712616515571, + 1.1686000021391967, + 0.16829196475783875, + 1.5808321538073473, + 2.70410552605102, + 3.6178371207276454, + 3.79689608866952, + 2.866628757066581, + 2.6770725110578164, + 1.8301948699739352, + 0.34330027045040296, + 1.6063924254120034, + 2.5823670655740942, + 3.436441437753598, + 3.2378092899819015, + 2.774749079562621, + 2.309619043161324, + 1.0268296678026332, + 0.7546533611494561, + 1.310773568236327, + 1.452897513945062, + 1.385879709152398, + 1.0724495764758515, + 1.5626300889323248, + 1.1006691166551759, + 0.9607477430529565, + 0.27847304409435747, + 1.3704352744501045, + 2.1628089184421597, + 2.2964729166266884, + 2.1897076608056207, + 1.7846556676119465, + 0.3440851545062355, + 0.9825156157532323, + 0.6161054486560682, + 0.36818937525941015, + 0.6805374054172546, + 1.6411524076550874, + 2.8714334705220717, + 2.36470107152306, + 1.819482958238095, + 1.600352766966237, + 0.6351773494050145, + 0.8140945808641146, + 1.145300706317083, + 1.1546384145310693, + 0.8342681417211963 + ], + "mean_band_dot": [ + 8.120597661864792e-06, + 5.9362413367125555e-06, + 1.4321613377887843e-05, + 8.382308095633562e-06, + 2.039934983599778e-06, + 1.9653542011610625e-06, + -1.183039375973749e-05, + -5.0252804157935316e-06, + -7.354535966896947e-06, + -1.0166705806113896e-05, + 2.1904808704675816e-06, + 3.3505137650990946e-05, + 1.1913604851088166e-05, + 6.614841390728543e-05, + 8.687356876180274e-05, + 8.82127337717975e-05, + 1.4059189140880335e-05, + 5.158366377600032e-06, + 8.950475091751287e-06, + 2.819076740934179e-06, + 2.072537313324574e-06, + 4.989042281522416e-07, + 2.340417040613829e-06, + -5.986470569041558e-06, + -8.24461062620685e-06, + -5.0956809332092234e-06, + 5.687106408913678e-06, + 2.715610310133343e-05, + 7.635724273313826e-06, + 7.835262795197195e-05, + 9.132768923336698e-05, + 9.283721669817169e-05, + 9.659864645072958e-05, + 9.655563007981982e-05, + 9.849037360254442e-05, + 9.282138989874511e-05, + 0.0001081455520761665, + 0.00010049411662294006, + 9.179197195408051e-05, + 9.577388527759467e-05, + 9.942122369466233e-05, + 9.778119206771407e-05, + 0.00010050393620986142, + 9.047870253198198e-05, + 9.350310301670106e-05, + 8.358513969142223e-05, + 9.055938789970241e-05, + 9.110200687700853e-05, + 9.442394366487861e-05, + 9.598698761692503e-05, + 8.717490868548339e-05, + 9.795665073397686e-05, + 0.00010885551773753832, + 8.625332088740834e-05, + 0.00010065039282380894, + 9.805256195249967e-05, + 9.571023474563845e-05, + 9.437422841074294e-05, + 9.636643676458334e-05, + 9.515400631698867e-05, + 9.33988933411456e-05, + 9.216913576892694e-05, + 9.605559125702712e-05, + 7.552297302027e-05, + 0.00010186239342147019, + 9.906431569106644e-05, + 8.660969160700915e-05, + 9.642286698863245e-05, + 9.427152372154524e-05, + 0.00010359561588302313, + 8.901369869818154e-05, + 0.00010521768035687273, + 0.00010003467355090834, + 9.620672153687337e-05, + 9.813899691835104e-05, + 9.646986700317939e-05, + 9.665364405009313e-05, + 9.645049999562616e-05, + 9.874726174530224e-05, + 0.00010093735545524396, + 9.555066026223358e-05, + 9.677254183770856e-05, + 9.659054217081575e-05, + 0.00010401417102912092, + 9.843570910561539e-05, + 8.974039792519761e-05, + 0.00010110727839673928, + 8.932881155487848e-05, + 9.848637819231953e-05, + 0.00010569418645900441, + 9.482244240643922e-05, + 8.867391170497285e-05, + 0.00010181781021856295, + 9.635882452130318e-05, + 9.685380655355402e-05, + 9.361357888337807e-05, + 8.484350291837472e-05, + 9.409923859493574e-05, + 9.174475530926429e-05, + 9.871853876575187e-05, + 8.448960647911008e-05, + 9.40111904128571e-05, + 9.456902421334235e-05, + 9.138207951764343e-05, + 9.884243218039046e-05, + 8.836141796564334e-05, + 9.935983234754531e-05, + 8.145340586906968e-05, + 9.56730417556173e-05, + 0.00010578270871519635, + 9.967230835172813e-05, + 8.659277955302969e-05, + 8.83523874790626e-05, + 9.304860532211023e-05, + 8.953833389568899e-05, + 0.00010324078994017327, + 9.00375041510415e-05, + 0.0001007491748623579, + 9.673931072029518e-05, + 8.504988636559574e-05, + 8.464021823328949e-05, + 8.995530242827954e-05, + 9.364306720271998e-05, + 0.00010242288271911093, + 8.669302513908406e-05, + 0.00010087674149872328, + 9.185679209622322e-05, + 9.816400370254996e-05 + ] + }, + { + "layer": 13, + "mean_snr": 0.8944227022852247, + "per_head_snr": [ + 0.9293150479167421, + 0.8494140218365209, + 0.8696755816032254, + 0.8908866234818149, + 0.9334674914165064, + 0.8846936544754844, + 0.8799239646301773, + 0.9180052329213255 + ], + "mean_band_energy": [ + 0.7187414638804981, + 0.7588553170486643, + 0.9665034266035, + 1.310735701209543, + 1.2629416505811402, + 1.4794112846820155, + 1.6987458429564546, + 2.1116313426606528, + 1.8714210924119445, + 1.9438881945689452, + 2.1368477731647033, + 2.375937690575187, + 2.42808957579847, + 2.64973198757367, + 2.600351756341066, + 2.6459115618857254, + 0.7839106429203673, + 0.8102268100493171, + 1.0417423563518988, + 1.3023300692192983, + 1.4171450494834001, + 1.5918958628227955, + 1.6915858138172073, + 1.9456772269589235, + 1.961287661629644, + 2.0272096975718625, + 2.125181144457505, + 2.263138620385508, + 2.498580727298938, + 2.549471972766108, + 2.6105505394236648, + 2.662247819064323, + 2.613505411593005, + 2.655126845118847, + 2.6358555475474272, + 2.574983159485411, + 2.6329394344936183, + 2.610457190651843, + 2.655207314080462, + 2.6483862504089455, + 2.5863547010097765, + 2.655376469841025, + 2.6462587566614264, + 2.64615586872171, + 2.544749909894609, + 2.643823448368817, + 2.637745982888209, + 2.621556146265659, + 2.681881862458628, + 2.597695152602201, + 2.6352717549191507, + 2.661639839361479, + 2.576993050363118, + 2.567705362016646, + 2.601932699630627, + 2.646889204633645, + 2.6386891471347056, + 2.6328371420192003, + 2.6173423708375, + 2.6228739629778133, + 2.5933405150355497, + 2.5916331328875373, + 2.637525861690138, + 2.6250897565146847, + 2.6045216841767918, + 2.6687335126014577, + 2.632261607485301, + 2.585668803654862, + 2.63105015498785, + 2.5201159928702612, + 2.638215306894571, + 2.6186772622949555, + 2.66108392549925, + 2.5992967180994437, + 2.480352728966732, + 2.6121297671615835, + 2.621060267644447, + 2.668509516229893, + 2.620568599336135, + 2.6364149603294837, + 2.6081240554189673, + 2.634219283722315, + 2.5399132563856064, + 2.645474228697557, + 2.604596193961429, + 2.6202602887336575, + 2.6304190135760184, + 2.6332327923530467, + 2.61907954257126, + 2.6013204204094045, + 2.6204303979962873, + 2.6225991317539474, + 2.6501853475612265, + 2.6483512075207845, + 2.6411812722040366, + 2.611776523766256, + 2.6156312896174967, + 2.6187902792890743, + 2.6380658497795366, + 2.6546795499244684, + 2.605788898606715, + 2.6566911921383145, + 2.6192351866619905, + 2.640442866454949, + 2.642958180801262, + 2.6440305853874797, + 2.6101936912247705, + 2.6321190137716464, + 2.5087515175682427, + 2.578972069211609, + 2.6470391406227556, + 2.511602153416595, + 2.54480518193491, + 2.6173661911442174, + 2.626802231367055, + 2.6245642026366234, + 2.638903604279971, + 2.6500502274860995, + 2.614806406252338, + 2.6141957596890695, + 2.6240694581517623, + 2.631164436420537, + 2.622184498815365, + 2.6361083154568057, + 2.628843280751015, + 2.582234211328796, + 2.629525492784012, + 2.6250548733344736 + ], + "dft_magnitude": [ + 309.64524452945113, + 23.217227928274458, + 17.276368951116513, + 8.348322411433205, + 0.2644989771813493, + 7.186569487489692, + 11.591096530755442, + 12.693491959454956, + 10.876667075890724, + 7.97398923115139, + 5.378870418654921, + 3.0643322495147984, + 0.24547673929752864, + 2.4476557993661006, + 5.019180777629059, + 6.385412934684546, + 6.290730342174681, + 4.989266757368167, + 4.043882341537286, + 1.7670698520855581, + 0.5415329195443879, + 1.0384657512341537, + 2.2215838245356054, + 3.225423564618108, + 3.69234655225938, + 3.123693380034851, + 2.4259815871088053, + 1.3314816832517338, + 0.37021075713291246, + 1.9209528382665584, + 2.8644319120050095, + 3.4367291512604954, + 3.569500221878326, + 2.940213155006587, + 1.9760217289832072, + 1.7438560745821636, + 0.5273647649485189, + 0.2521294779175139, + 1.550530692984492, + 2.024836242012076, + 1.795707776880586, + 1.6202714866248844, + 1.167131957897436, + 0.5898545432374201, + 0.537745166197499, + 0.9175717304877978, + 1.7731593651739772, + 2.1907481247924787, + 1.8297503120057683, + 1.9923726487335802, + 1.3074733605845266, + 0.5047177995367492, + 0.670143824793088, + 0.5053061770934234, + 1.335167865229578, + 1.5605742078276412, + 1.5255537553357545, + 1.1000746379671922, + 1.5074013328341678, + 0.47716248305412956, + 0.6035599327428297, + 1.2793230473949573, + 1.8492333280704916, + 2.267091178703713, + 2.9055083269641955 + ], + "mean_band_dot": [ + -3.9865615235612495e-06, + -1.1039382414423926e-05, + 3.4237558850236383e-06, + -2.33102019819853e-06, + 1.99560673763699e-06, + -1.4211673942554626e-05, + -2.4976524564124247e-05, + -3.174229149749408e-05, + -2.3697434812675056e-05, + -1.558167014081846e-06, + -4.607555069924274e-06, + 1.4399633982975502e-05, + 2.096422633712791e-05, + 4.992951426174841e-05, + 6.290611963777337e-05, + 7.316205551433086e-05, + -9.129828129061934e-06, + -1.272556659159818e-05, + 5.472615242751999e-06, + -4.986844089671649e-06, + 2.663900431798538e-06, + -7.879588622472511e-06, + -3.1646628713133396e-06, + -1.7246646734747628e-05, + -2.3732268573439796e-05, + -4.6738498440390686e-06, + 3.697129500324081e-07, + 5.115785825182684e-06, + 1.3381165445025545e-05, + 4.999572490760329e-05, + 6.651994004869266e-05, + 7.849752955735312e-05, + 7.504711743422376e-05, + 8.121436621877365e-05, + 6.97709195378593e-05, + 6.39450668131758e-05, + 7.691368318774039e-05, + 7.355303955591808e-05, + 7.762004088363028e-05, + 7.080372006385005e-05, + 7.128371976250492e-05, + 7.219562553473224e-05, + 7.35197843368951e-05, + 6.978182494776775e-05, + 6.537995147937181e-05, + 6.486055352183939e-05, + 6.955742344416649e-05, + 6.636095042722445e-05, + 6.104616647917283e-05, + 6.444923997150909e-05, + 7.379515557204286e-05, + 7.421245163641288e-05, + 6.631237144461011e-05, + 6.866643991543242e-05, + 5.9636648302330286e-05, + 7.001719507115922e-05, + 6.866181445275288e-05, + 7.208373688172287e-05, + 7.582122304938821e-05, + 7.440354522714188e-05, + 6.585476259601819e-05, + 6.654359242475039e-05, + 7.077394616317179e-05, + 6.803628775742254e-05, + 7.71937923218502e-05, + 6.796705633860256e-05, + 7.151471095312445e-05, + 6.355262036095155e-05, + 7.157556387937802e-05, + 4.866250833401864e-05, + 7.945225956973445e-05, + 6.734142220921058e-05, + 8.130833657560288e-05, + 6.960799578337173e-05, + 5.5176558134917286e-05, + 7.641145427328411e-05, + 7.02202285083331e-05, + 7.236636793095386e-05, + 7.025163085927488e-05, + 7.285772267096036e-05, + 6.728381322318455e-05, + 6.853375396076444e-05, + 6.298611788224662e-05, + 7.449665247349913e-05, + 6.783851586078526e-05, + 6.859332967223963e-05, + 6.692269266750372e-05, + 7.056491699586331e-05, + 6.95197417712734e-05, + 6.842770346793259e-05, + 6.695767490327853e-05, + 7.720252233411884e-05, + 7.429166544170585e-05, + 6.993939894073264e-05, + 7.008008140019228e-05, + 7.150351473228511e-05, + 6.595744434889639e-05, + 7.645224422958563e-05, + 6.144425890397542e-05, + 7.16153624580329e-05, + 7.299900786961189e-05, + 7.128932134037314e-05, + 6.680212175069755e-05, + 6.607954287574103e-05, + 7.440639228661894e-05, + 7.218300247302523e-05, + 7.124759878252007e-05, + 7.270694709404779e-05, + 5.429985446880892e-05, + 5.4176242656467366e-05, + 7.60464971563124e-05, + 4.461906314645603e-05, + 6.887215977258165e-05, + 5.713571584919919e-05, + 7.533894040534506e-05, + 7.486909066756198e-05, + 7.417727101710625e-05, + 7.349474890361307e-05, + 6.241858691424795e-05, + 6.651471733221115e-05, + 7.748926662998201e-05, + 6.378603234225011e-05, + 7.172106592179261e-05, + 6.619466188340084e-05, + 7.203733974847637e-05, + 6.307276589723187e-05, + 7.01372557614377e-05, + 6.411934691641363e-05 + ] + }, + { + "layer": 14, + "mean_snr": 0.9209576973621747, + "per_head_snr": [ + 0.9293270577555497, + 0.9206360641798387, + 0.9706139578764451, + 0.9149255115436501, + 0.9200263448282879, + 0.9190149493326825, + 0.8775412731728616, + 0.9155764202080818 + ], + "mean_band_energy": [ + 1.2247371562267775, + 1.1835106574489642, + 1.565602742056985, + 2.0277805616967424, + 2.050772826998543, + 2.2876983029936007, + 2.506432291144291, + 2.877522839943774, + 3.006728360878048, + 2.848534603567604, + 2.8361500353295224, + 3.161374366599894, + 2.8380665929364497, + 3.1740715775670836, + 3.0700360027478695, + 3.190514506490251, + 1.1596119488031618, + 1.2329753162144748, + 1.5358849098121947, + 2.02162547310839, + 1.9970871628938678, + 2.3985858826305098, + 2.4612497903185346, + 2.8487976145273404, + 2.922925759949697, + 3.1178562719937286, + 2.675081557540258, + 2.8323729660180597, + 2.925710180214546, + 3.120630142364057, + 3.2314979173324883, + 3.2345937742200555, + 3.2023702218419086, + 3.258408836694418, + 3.219354092873255, + 3.2362522008225945, + 3.206174117867567, + 3.222941367334503, + 3.2255946241399496, + 3.227415380339382, + 3.2318529295554033, + 3.221189658671804, + 3.2417664414280782, + 3.173476151342795, + 3.20862003751963, + 3.1909044370314987, + 3.2350851263478777, + 3.2383367790673514, + 3.1846359965822284, + 3.2081793779886123, + 3.2209972989003486, + 3.256596635593292, + 3.232889515601494, + 3.2273186385230463, + 3.2214279636920846, + 3.2526663239882936, + 3.131331500578929, + 3.2208410634383426, + 3.189913382293705, + 3.2384310289176437, + 3.203648259822854, + 3.241238018828387, + 3.2267838250551524, + 3.133885505579629, + 3.252189554020825, + 3.158266099456924, + 3.2110717321435267, + 3.201093085336863, + 3.212343751656931, + 3.227227550054577, + 3.2092394999004537, + 3.206739024653297, + 3.2408482286036993, + 3.21645131438207, + 3.1874398689583607, + 3.2264571280034247, + 3.2104112310118644, + 3.2100131593498453, + 3.2063245567598617, + 3.216048729893517, + 3.2359350176987363, + 3.2012897309209247, + 3.2431671460022375, + 3.236092450226492, + 3.2089574169603488, + 3.246888287429842, + 3.1845833326984767, + 3.23087251110457, + 3.2129072930921794, + 3.1850791400845253, + 3.1919257041373132, + 3.1696282095097175, + 3.196320822400761, + 3.1934076064349544, + 3.281383716082802, + 3.207095989449275, + 3.2485292292481915, + 3.2035625090724835, + 3.2073422617771126, + 3.212174669368439, + 3.2091119985947554, + 3.209241662373932, + 3.2453648201273264, + 3.2387915411542583, + 3.239505718978995, + 3.2075082719573995, + 3.1883050606528602, + 3.220841452071948, + 3.224232317730463, + 3.211219621790846, + 3.2091458464023788, + 3.197040761226811, + 3.246529044690625, + 3.2056271792863846, + 3.2563291800774703, + 3.1818189920539908, + 3.2378025184591728, + 3.2054194562773866, + 3.190864113277005, + 3.201297682948793, + 3.234311105836648, + 3.2096374669961136, + 3.1875126568047865, + 3.2364115226160823, + 3.1681095239660486, + 3.201677311659733, + 3.2586013911167875, + 3.25755726097594 + ], + "dft_magnitude": [ + 388.26566986882017, + 21.12309144006287, + 15.731539639594283, + 7.9048277179803925, + 0.32041657763047804, + 6.3797085299019995, + 10.663673816172867, + 12.011434566654078, + 11.330271479737156, + 10.160357598024838, + 7.544082520425335, + 4.0922677716038836, + 0.6744704114989837, + 3.27493614998374, + 5.940977931822175, + 6.448535642596715, + 6.454124725147368, + 4.762491162495133, + 2.7858737688960358, + 1.8683570826264575, + 0.5373109563800383, + 1.2281329767972324, + 2.7893346643104864, + 4.3848714340336326, + 5.148953589419576, + 4.543178328874073, + 3.2279441697230244, + 1.5063969005323137, + 0.6676563948252301, + 2.313315351578171, + 2.8600979017931216, + 3.131890149020156, + 3.459349450283252, + 3.295731571206536, + 2.461120898634359, + 1.9177216129539802, + 0.5461698266777295, + 0.7310197786950651, + 1.8818429111930886, + 2.6908516661460657, + 2.623779221655494, + 2.219724325137491, + 1.5401900262372106, + 0.6362945604904825, + 0.24289464577281292, + 0.8309430986935255, + 0.9621545777035141, + 1.1543332055468645, + 1.1297565927195095, + 1.5428675638505311, + 0.76534123938953, + 0.5207776846448356, + 0.8696552534019767, + 1.4111031452545078, + 1.325575149661765, + 1.9340946505916157, + 1.2133554978269407, + 1.970013299028485, + 1.492680346835102, + 0.8812146193675914, + 0.7406746219095673, + 1.2588695602025555, + 2.4361891102145523, + 3.3412421943962016, + 3.412337410514766 + ], + "mean_band_dot": [ + -4.9799644955328404e-05, + -5.26440925341376e-05, + -5.358791725029732e-06, + -2.7353878294889e-05, + -8.597876956173423e-06, + -1.8917757529379742e-05, + -1.938484038532806e-05, + -3.0451277496013063e-05, + -4.544847945453512e-05, + -1.1655652144781925e-05, + -1.0674181453396159e-05, + -6.4886891095738974e-06, + -3.436317888372287e-05, + 5.42733793622574e-05, + 2.6938040377899597e-05, + 5.3363192478173005e-05, + -5.539660185149842e-05, + -5.771998667114531e-05, + 1.9724257072084583e-07, + -3.373870896439257e-05, + -1.9789523150848254e-05, + -1.999457765577972e-05, + -3.628578872394428e-05, + -5.0252612140866404e-05, + -2.8354475432479376e-05, + -3.140854943239901e-05, + -9.812730695557548e-06, + 6.134841896709986e-06, + -2.7598943574957957e-05, + 3.635255660583425e-05, + 5.618983200861294e-05, + 5.8789752245047566e-05, + 6.272028417697584e-05, + 5.910803753295113e-05, + 5.368345074430181e-05, + 5.738288655265933e-05, + 6.118028522905661e-05, + 4.23496379653443e-05, + 5.9400411259957764e-05, + 5.1375309453760565e-05, + 5.5176952912461275e-05, + 5.350390301828156e-05, + 5.408760807767976e-05, + 6.199075210133742e-05, + 5.228366865139833e-05, + 4.9910350128357095e-05, + 5.572926119157273e-05, + 5.4301982416404826e-05, + 5.4133659205035656e-05, + 5.2278384117698806e-05, + 5.592216945160544e-05, + 6.251771310417098e-05, + 5.729251279262826e-05, + 5.036374477640493e-05, + 5.761821967098513e-05, + 5.767578051063538e-05, + 4.737418662159598e-05, + 5.620526644634083e-05, + 5.279871993479901e-05, + 5.801574093311501e-05, + 5.589194825006416e-05, + 5.579689059231896e-05, + 5.295829242868422e-05, + 5.865483740308264e-05, + 5.512030625709485e-05, + 3.771217828330009e-05, + 4.651409224720737e-05, + 5.61829805292291e-05, + 4.981358898703547e-05, + 4.8084705554174434e-05, + 4.796560850195419e-05, + 5.694623450835934e-05, + 5.962811415827218e-05, + 4.328919220597527e-05, + 4.996493817088776e-05, + 4.541071211860981e-05, + 6.121470539710572e-05, + 5.414158943040093e-05, + 5.170504817897381e-05, + 5.527316443476593e-05, + 4.79139284266239e-05, + 4.8341461649670237e-05, + 5.3155227988099796e-05, + 6.029315147770831e-05, + 5.72261144498043e-05, + 4.9158409467509045e-05, + 5.1758298155846205e-05, + 5.176473121082381e-05, + 5.38605911515333e-05, + 4.969398213461318e-05, + 5.424473988568934e-05, + 4.816181910882733e-05, + 5.409952787260863e-05, + 5.1243412599433213e-05, + 5.5557554475171855e-05, + 5.582229277933948e-05, + 4.362161013204968e-05, + 5.173786928480695e-05, + 5.729141730625997e-05, + 5.160527177849872e-05, + 5.872490123692842e-05, + 4.887130512543081e-05, + 5.5880942966268776e-05, + 5.186308919746807e-05, + 5.3132363802887994e-05, + 5.2821999105390205e-05, + 5.85209608061632e-05, + 5.120399470115444e-05, + 5.813578826519006e-05, + 5.091072119967066e-05, + 5.5533933732476726e-05, + 5.3082244676261325e-05, + 4.984495012649859e-05, + 6.024521923109205e-05, + 6.104272665652388e-05, + 5.2166763907735e-05, + 5.177558119839887e-05, + 5.503031147213733e-05, + 6.165936906654679e-05, + 5.471822368008361e-05, + 5.397765983161662e-05, + 5.42564141596813e-05, + 5.3592853021200426e-05, + 5.2854431260129786e-05, + 4.8736146027295035e-05, + 5.0412584698733554e-05, + 6.160073331784588e-05, + 5.1323842114925355e-05 + ] + }, + { + "layer": 15, + "mean_snr": 0.9131732829829318, + "per_head_snr": [ + 0.9335009864161137, + 0.9283846472304403, + 0.9138014615881804, + 0.8799273309371092, + 0.9192097160556576, + 0.9179337942818249, + 0.8714285929579031, + 0.9411997343962253 + ], + "mean_band_energy": [ + 1.5031938731842622, + 1.4635260352443105, + 1.572663227777733, + 1.9929858101765214, + 1.9053381316101028, + 2.2477792982163525, + 2.3537039836601394, + 2.4704475169623628, + 2.8323844071474937, + 2.641900281741691, + 2.8105147749984596, + 2.853946949107474, + 3.0065796912447507, + 3.2374534938425796, + 3.342492200816805, + 3.3795158664937706, + 1.4972763031899259, + 1.499775369605489, + 1.8020880678970777, + 2.0721684500220157, + 1.8848991629939595, + 2.182235718224403, + 2.380411726867493, + 2.596937049534649, + 2.735096945482695, + 2.694787618099035, + 2.7763052001499897, + 3.1542754833359634, + 3.059520991556159, + 3.199205381580537, + 3.355753779756112, + 3.348057790353664, + 3.331572914749124, + 3.3735229273504643, + 3.3403287842740337, + 3.3170259625171568, + 3.3627856316663784, + 3.3230422669706687, + 3.341233331674415, + 3.3374025248920915, + 3.3076821237857867, + 3.3504053483050775, + 3.3038809687134467, + 3.360222490075085, + 3.3597120030346144, + 3.3618532939741197, + 3.331156632461406, + 3.344468494299621, + 3.341328473825582, + 3.3449798039239305, + 3.3134521922577616, + 3.366130385527361, + 3.3811209411813117, + 3.3442776747421767, + 3.3148599932635676, + 3.252133281524179, + 3.3360685913713546, + 3.362657124440142, + 3.3581040350160514, + 3.3589804087322417, + 3.21194513248531, + 3.374132000786334, + 3.301213089132892, + 3.332085722285557, + 3.3743265370991917, + 3.3496898607803782, + 3.291392782220317, + 3.3587550611741737, + 3.303391290164761, + 3.3503905058319, + 3.3581443943100453, + 3.324264688746883, + 3.368099130759912, + 3.2769569953737534, + 3.321316643091283, + 3.360225206339404, + 3.3422618306335714, + 3.3292494920282234, + 3.34563651215265, + 3.289250035135769, + 3.3619971698362434, + 3.364916757445494, + 3.3436509264703727, + 3.3313392111412448, + 3.3982215407333616, + 3.3424485560168833, + 3.364240139267915, + 3.3650673007496827, + 3.3763272636706265, + 3.3526154683612432, + 3.364640497318039, + 3.3367623948134195, + 3.3452462018519, + 3.315252487500045, + 3.3271894716852053, + 3.3440258391350746, + 3.357949240438167, + 3.350357921227321, + 3.3497973702367148, + 3.3315739339589765, + 3.3250331671472466, + 3.231587034355286, + 3.342092227248685, + 3.3113615981649733, + 3.2362748218313815, + 3.3316647485937523, + 3.3471876107255127, + 3.3614903713500173, + 3.3583260851434353, + 3.3393378553699673, + 3.3577203627499053, + 3.3743952439343428, + 3.3829632754513064, + 3.320773868568404, + 3.3351944785454624, + 3.316334777735756, + 3.3095706453771054, + 3.333337323637714, + 3.3608840215626703, + 3.2977884722554975, + 3.325278494672709, + 3.300025878096257, + 3.3378019717061953, + 3.190047042176472, + 3.3400173706604193, + 3.345507768940495, + 3.2845313981069353, + 3.325370453224835 + ], + "dft_magnitude": [ + 399.9818541551461, + 24.794093043858584, + 17.985460980556034, + 9.242865719207357, + 0.7125401562820162, + 7.503441292499235, + 10.982521320100386, + 12.236534264841861, + 10.751898207966486, + 8.383632438031317, + 5.278014844272178, + 2.2141268565687193, + 0.1334465684568955, + 2.5662677714744246, + 4.870660385746346, + 5.855159647536271, + 5.698776465913667, + 5.097722836231, + 3.353038796632808, + 1.4218544496923207, + 0.42494406867016815, + 1.9400681651287808, + 2.4135673912954085, + 3.6704930301283554, + 3.824690557896048, + 3.7861043389439213, + 3.4441887321006472, + 2.049903605561685, + 0.24212176365454732, + 1.069360418639519, + 2.321486573642078, + 3.153551296129682, + 3.136075374867261, + 2.7845055600727706, + 1.5573664530594604, + 0.9637187695514914, + 0.18224720371044084, + 0.8892889632489672, + 1.2496760686328123, + 2.037143691311084, + 2.174399136119019, + 2.8850413593153377, + 2.126594011011317, + 0.7258571879655754, + 0.2583778833056004, + 1.1507481768304033, + 0.8953681392145455, + 1.574327986355428, + 0.749962367235877, + 1.3241445383058765, + 0.7594124977079574, + 0.5981197396773628, + 0.13629088366051859, + 1.363879014458176, + 1.9507723897709, + 2.2060670228613577, + 2.360306024871285, + 1.781192713668838, + 1.9204182907495004, + 0.8667784337844625, + 0.41590338124066106, + 0.7198385864260871, + 1.3030678699322633, + 1.9549287723058884, + 1.9991097949552454 + ], + "mean_band_dot": [ + 1.711930235615e-05, + 1.9797373695951137e-05, + 3.618121773740768e-05, + 1.7562880330501685e-05, + 1.9627168967417674e-05, + 1.949694024006021e-06, + -3.936908910873171e-06, + 2.0455547797837426e-06, + 6.056995403014298e-06, + 2.2610781229559507e-05, + 1.2646706466057367e-05, + 4.275218179827789e-05, + 3.2483748782397015e-05, + 8.141379976223106e-05, + 9.718797173263738e-05, + 0.00010994539343300858, + 2.8588545660568343e-05, + 1.4328663837659406e-05, + 6.0522695463305354e-05, + 2.434957917785141e-05, + 1.97001141657438e-05, + 1.6693414266910622e-05, + 1.8947428827686963e-05, + -9.61690830081352e-07, + 5.165270522411447e-06, + 1.2950975019521138e-05, + 2.5506904137273523e-05, + 3.972161471210711e-05, + 2.9604268888761e-05, + 6.50135239084193e-05, + 8.946186244429555e-05, + 9.799262170417933e-05, + 0.00010141783559447504, + 0.00011377391274436377, + 9.991351589633268e-05, + 0.00010842320125448168, + 0.00010329439101042226, + 0.00010087266400660155, + 0.00010207005880147335, + 9.47216276472318e-05, + 9.382097823618096e-05, + 9.938267794495914e-05, + 0.00010342488531023264, + 9.854543623077916e-05, + 0.00010015455700340681, + 9.40559211812797e-05, + 0.0001035194100040826, + 0.00010986459710693453, + 0.00010049860884464579, + 0.00010504892998142168, + 9.19701578823151e-05, + 9.502216471446445e-05, + 0.00010593913793854881, + 9.554898815622437e-05, + 9.655480789660942e-05, + 9.602554291632259e-05, + 9.53607882365759e-05, + 0.0001109130334953079, + 0.00010069447898786166, + 9.060231468538404e-05, + 8.149682253133506e-05, + 0.00010693085368984612, + 8.446008223472745e-05, + 0.00010464169758961361, + 0.00010744435030574095, + 0.0001045083536155289, + 9.195543952955632e-05, + 9.971049667001353e-05, + 0.00010362393368268386, + 0.0001006445368147979, + 8.912024213714176e-05, + 0.00010164211153096403, + 0.00010735148498497438, + 9.381440622746595e-05, + 0.00010641117205523187, + 9.997897359426133e-05, + 9.268019721275778e-05, + 0.00010412630899736541, + 0.00010042304802482249, + 9.116592400459922e-05, + 0.00010779346212075325, + 0.00010485424945727573, + 9.881680944090476e-05, + 0.00010011753192884498, + 9.636271533963736e-05, + 0.00010035221293946961, + 0.00010591696627670899, + 0.00010420304533909075, + 0.00011006279146386078, + 9.556345457895077e-05, + 0.0001040089287016599, + 0.00010039552489615744, + 0.00010552716958045494, + 0.00010000190604841919, + 9.503472392680123e-05, + 0.00010445056886965176, + 0.00010619222575769527, + 0.00010331550583941862, + 8.98217754183861e-05, + 9.75866278167814e-05, + 9.626987457522773e-05, + 0.00010133223304364947, + 9.824611970543629e-05, + 0.0001013418964248558, + 8.963476966528106e-05, + 0.0001101442908293393, + 9.705661068437621e-05, + 0.00011043773247365607, + 0.00011512846458572312, + 0.00010125098151547718, + 0.00010204113186773611, + 0.00010437411310704192, + 0.00010254780181639944, + 9.817868522077333e-05, + 0.0001045472845362383, + 9.755903329278226e-05, + 9.909751224768115e-05, + 0.00010029099053099344, + 0.00010140314225282054, + 9.559043519402621e-05, + 9.593940967533854e-05, + 8.467462794214953e-05, + 0.000104628080407565, + 8.646028641123849e-05, + 9.632603087084135e-05, + 0.00010121632021764526, + 9.656458769313758e-05, + 9.784151188796386e-05 + ] + } + ], + "elapsed_s": 1.0560877323150635 +} \ No newline at end of file diff --git a/data/exp_wqk_spectral/pythia-2.8b_wqk_spectral.json b/data/exp_wqk_spectral/pythia-2.8b_wqk_spectral.json new file mode 100644 index 0000000000000000000000000000000000000000..39dd3ed93f54869c3906d0a9215124d44f9f0728 --- /dev/null +++ b/data/exp_wqk_spectral/pythia-2.8b_wqk_spectral.json @@ -0,0 +1,4655 @@ +{ + "model": "EleutherAI/pythia-2.8b", + "short_name": "pythia-2.8b", + "theta": 10000, + "T_train": 2048, + "d_head": 80, + "n_pairs": 40, + "k_dead": 29, + "global_mean_snr": 0.8730842178773597, + "global_min_snr": 0.8133661844037099, + "global_max_snr": 0.9160213598809545, + "layers": [ + { + "layer": 0, + "mean_snr": 0.9052348412152698, + "per_head_snr": [ + 0.8958706432307164, + 0.9986665767624794, + 0.8835039016761658, + 0.9844128319969461, + 0.9240449231617176, + 0.884039775534908, + 0.9014942567334184, + 0.8091113151055305, + 0.9249134910942546, + 0.8696329054804117, + 0.9720918210344004, + 0.9365686040296097, + 0.9130597528924688, + 0.9468180693805767, + 0.8524345601182877, + 0.8282880283122332, + 0.9155505341358355, + 0.9204775158028794, + 0.934427027998395, + 0.9256076503543281, + 0.8746850227113879, + 0.9085658843745771, + 0.8810773922490648, + 0.9016429430468095, + 0.9041604464715586, + 0.899928224250187, + 0.8507703761481246, + 0.893805403443798, + 0.9143945122467159, + 0.8722331755086927, + 0.9439359532652266, + 0.9013014003369274 + ], + "mean_band_energy": [ + 0.4576604512947703, + 0.41278982004836556, + 0.44407678142231766, + 0.6693591110675885, + 0.6584232609195838, + 0.46104250315782247, + 0.41392818508510953, + 0.42417349789194575, + 0.6307012560493248, + 0.6816023719565809, + 0.731894007517753, + 0.7366725456190106, + 0.7347622913378754, + 0.7372709764234427, + 0.7478153449726922, + 0.739157339197168, + 0.741327699862727, + 0.7375045126692894, + 0.74492169907673, + 0.746550961830988, + 0.7380923332274668, + 0.7415837308472304, + 0.738372763089511, + 0.7369893093076042, + 0.7391616101821539, + 0.7419582174554916, + 0.7355242029528752, + 0.7435047688042051, + 0.736765980767225, + 0.7437458789783425, + 0.7392376608674278, + 0.7387118170731636, + 0.7375593950274847, + 0.7354320917813628, + 0.7329678358599715, + 0.7371223115414923, + 0.7402569938774439, + 0.7357693912421159, + 0.7487189080108915, + 0.7394707372099003 + ], + "dft_magnitude": [ + 27.432580555504444, + 1.955171959536307, + 1.4018263315173938, + 0.6760031379356449, + 0.015653041061436706, + 0.569360795654558, + 0.8614151905953832, + 0.9082134391171578, + 0.7823732182035017, + 0.5291668461764684, + 0.31453923916600673, + 0.15383375690262144, + 0.05463222887122496, + 0.06159545781195804, + 0.04564431214486772, + 0.1100634218864948, + 0.21497454293589585, + 0.2597724774152077, + 0.23000917169101187, + 0.16300447796810677, + 0.04824323270177722 + ], + "mean_band_dot": [ + -4.869338448543204e-06, + -7.938121797224085e-07, + 1.0767815375345435e-06, + 3.0676013693664564e-07, + -8.425368230291495e-06, + -6.153622541660297e-06, + -1.7308997399823185e-07, + 9.244711748124244e-07, + 3.099039719245411e-06, + -7.589841874278135e-06, + -7.919377306109252e-06, + -8.882005283794568e-06, + -9.800905691292884e-06, + -8.593476951546107e-06, + -8.514730102149313e-06, + -9.541335892038204e-06, + -8.516299885741319e-06, + -8.41796845634235e-06, + -1.1141822889726429e-05, + -8.32750712049801e-06, + -8.58272258597026e-06, + -9.351309303440303e-06, + -8.100332951244128e-06, + -8.517454918433032e-06, + -8.657401417622167e-06, + -7.75890142250546e-06, + -7.869252985415188e-06, + -8.019235698952798e-06, + -7.813363913555804e-06, + -8.751693090403024e-06, + -8.131346527306958e-06, + -8.499286263941032e-06, + -8.633393591139793e-06, + -8.760494121418108e-06, + -8.46315484466231e-06, + -8.013554140973155e-06, + -8.503499640255543e-06, + -8.811579435530348e-06, + -9.16741222596329e-06, + -9.006929275301447e-06 + ] + }, + { + "layer": 1, + "mean_snr": 0.9129063937945933, + "per_head_snr": [ + 0.8295960704520214, + 0.8557741207045647, + 0.8935309842019822, + 0.9129957958319408, + 0.9160493147855898, + 0.920482574583182, + 0.9224567849622908, + 0.9733303775594151, + 0.938589271123421, + 1.1285914654719673, + 0.9027720845166268, + 0.9237822967823558, + 0.8760610885419445, + 0.8915817638009891, + 0.8510931714618098, + 0.8382221088893484, + 0.8596898165887179, + 0.8767502643538803, + 0.9307775893035254, + 0.9874106887948351, + 0.8948823594288685, + 0.9309528529180715, + 0.9448591821872862, + 0.9063099973451678, + 0.9565682056987022, + 0.8802072251339326, + 0.9005307115003559, + 0.9706308640498831, + 0.8706351683954131, + 0.9156214162740931, + 0.9227260454510703, + 0.8895429403337312 + ], + "mean_band_energy": [ + 1.3149303820367417, + 1.1480094517566835, + 1.2533119537197277, + 1.3271205376272697, + 2.0047165719703064, + 1.2946692177650307, + 1.122347928423122, + 1.2002744501612295, + 1.474952594247313, + 1.9302533255426204, + 2.026856642245484, + 2.036279546008765, + 2.047138902792943, + 2.0139647358534387, + 2.0231063726124083, + 2.004594159794979, + 2.0100063690247403, + 2.0440926618104527, + 2.0317523333209726, + 2.030735399692809, + 2.0314232827662657, + 2.0150095861373787, + 2.0246532245776643, + 2.014566756484097, + 2.0291517911062664, + 1.9944618527243092, + 2.0120931217621316, + 2.0146785437311037, + 2.0237115464175988, + 2.007901518047186, + 1.9919682843166395, + 2.0325863923143537, + 2.0277163147681665, + 2.009267345023847, + 2.0403723643599383, + 2.0408576599028057, + 2.013254552212821, + 2.0323398273009734, + 2.0214592270496583, + 1.9947739009568592 + ], + "dft_magnitude": [ + 74.7113606283671, + 5.55272076345409, + 4.178880121417537, + 2.0848500584366554, + 0.059282626995585036, + 1.5828144004837295, + 2.3225600822315196, + 2.267523698435267, + 1.7605819646269445, + 1.0634844867431674, + 0.4761845563436561, + 0.03258360113746155, + 0.3111141958239493, + 0.5040600591989582, + 0.6776719815271101, + 0.9738219251220139, + 1.0787522420931035, + 0.7790051052820786, + 0.6474925536656905, + 0.36064196317923536, + 0.33848689109471763 + ], + "mean_band_dot": [ + -1.8571194107153133e-05, + -2.6871658178606693e-07, + -7.886391126987747e-06, + -1.4039114012120991e-05, + -3.4505756195812866e-05, + -1.761442629515386e-05, + 2.0498328851203947e-06, + -5.182171994988494e-06, + -1.0185002739149241e-05, + -3.469364397972185e-05, + -3.3878640471129985e-05, + -3.434540582532008e-05, + -3.702392691309342e-05, + -3.570864496680315e-05, + -3.56131144229721e-05, + -3.408603587615744e-05, + -3.408324416227515e-05, + -3.239464007265269e-05, + -3.6975172093889335e-05, + -3.274968346573814e-05, + -3.302568084109225e-05, + -3.421752941363821e-05, + -3.4227513904738774e-05, + -3.1178772854900674e-05, + -3.283136529148578e-05, + -3.4337268550643786e-05, + -3.3788337378837245e-05, + -3.169005953438387e-05, + -3.378974208771979e-05, + -3.280689251425884e-05, + -3.0856745530627444e-05, + -3.402078820613496e-05, + -3.279607838635457e-05, + -2.9885602191370712e-05, + -3.516750805374614e-05, + -3.510544399603078e-05, + -3.229217482498825e-05, + -3.4305185437233376e-05, + -3.4557245382416116e-05, + -3.337357590709189e-05 + ] + }, + { + "layer": 2, + "mean_snr": 0.9021332000250302, + "per_head_snr": [ + 0.8819467585328543, + 0.8937851277361479, + 0.9311255406848021, + 0.9071724303720637, + 0.8885641898029824, + 0.9585936331810779, + 0.9046669651722629, + 0.9465848248513281, + 0.923407391600567, + 0.9383282963092059, + 0.9138188067196512, + 0.9495009231618475, + 0.8704832514208015, + 0.8601687945826957, + 0.9126698024977011, + 0.8725048061350285, + 0.8637315528593034, + 0.9163926728704298, + 0.8019355368691908, + 0.9086910625708121, + 0.9215781484455337, + 0.9017725604493457, + 0.9149169941902376, + 0.8651685957571602, + 0.9152308370292058, + 0.8468341069049548, + 0.9300559530475653, + 0.8806537623076517, + 0.9197289272253749, + 0.8998204010015617, + 0.9277004051414025, + 0.9007293413702157 + ], + "mean_band_energy": [ + 1.4240414375631225, + 1.3475029187673793, + 1.383444559205129, + 1.418432725292256, + 2.031363556698947, + 1.4417018910467805, + 1.3996723582685093, + 1.2708037521670124, + 1.392689702102106, + 1.9933391231459439, + 2.2078022102059287, + 2.1875647094873987, + 2.194358567056094, + 2.1621338564325536, + 2.219672157749754, + 2.186393652676583, + 2.177461907780165, + 2.203766200873145, + 2.203310450841044, + 2.1862879473750088, + 2.178516734691852, + 2.150718690910727, + 2.20231016918115, + 2.1604784100751107, + 2.190832978294554, + 2.1752029467538945, + 2.17966770956836, + 2.1791450868189433, + 2.1660575154936685, + 2.17601937326001, + 2.2101327610774377, + 2.147650627223916, + 2.179643002592058, + 2.154825588617348, + 2.214330429021496, + 2.1686704644201917, + 2.1155536594202724, + 2.185645159405592, + 2.1575186915483853, + 2.1935563969081393 + ], + "dft_magnitude": [ + 80.51822008001797, + 6.05716395264911, + 4.381252377328563, + 2.0390116866215804, + 0.13431880587858477, + 1.6207280723156177, + 2.394923977898015, + 2.175862752139483, + 1.5507385028346035, + 0.5350823007313364, + 0.3447386533924275, + 0.25770381088525657, + 0.024291284749745047, + 0.3507011328484017, + 0.7480379049208502, + 1.0230757774132058, + 1.2231070657541312, + 0.9629986942079385, + 0.6279396960495803, + 0.25909808751167046, + 0.3385410367020967 + ], + "mean_band_dot": [ + -6.491270737285503e-06, + 6.880480877669016e-06, + 6.916877308071889e-06, + 1.4436464698519557e-05, + 4.7586616341277475e-05, + -1.1946335419565914e-05, + 1.0652326278659529e-05, + 1.0163312282429616e-06, + 4.316456158903748e-06, + 3.9761855356346144e-05, + 6.277828989311731e-05, + 6.723028872102077e-05, + 5.710281515121097e-05, + 5.971843537508902e-05, + 6.21542069836778e-05, + 5.958176916465164e-05, + 5.845199494274311e-05, + 6.712696774684445e-05, + 6.535207239721785e-05, + 6.65246555001886e-05, + 6.03495719900593e-05, + 5.849458943885111e-05, + 6.303174283175394e-05, + 5.811877251176157e-05, + 6.330654351813793e-05, + 6.277601637521001e-05, + 6.395561445557973e-05, + 6.337062781653914e-05, + 6.498994858930018e-05, + 5.600463942130318e-05, + 6.1244367319091e-05, + 5.818695777861649e-05, + 6.399061346655798e-05, + 5.882189733483755e-05, + 6.442362702614446e-05, + 6.17728214933777e-05, + 5.583893012044427e-05, + 6.444359370902929e-05, + 5.987994577481003e-05, + 6.184131656254975e-05 + ] + }, + { + "layer": 3, + "mean_snr": 0.8993515213055205, + "per_head_snr": [ + 0.870259219234786, + 0.8805281771558097, + 0.9245698388774534, + 0.8670309336403429, + 0.9304348597072709, + 0.8624487013192665, + 0.9454955362086432, + 0.9248975590377156, + 0.9622535001070274, + 0.8995722223895722, + 0.8879585005807294, + 0.7821827154021325, + 0.8497153518092907, + 0.8775624066874483, + 0.8760680546585287, + 0.8881115189565079, + 0.9263596582619692, + 0.8244349960510561, + 1.0782845587971295, + 0.866457614833588, + 0.8736480088445506, + 0.9324012209042842, + 0.8547189192090702, + 0.9113648877218018, + 0.8799585501959184, + 0.9473320488391932, + 0.9161665040506334, + 0.9695281936834178, + 0.8703895054319862, + 0.8784716554361085, + 0.9059132470197743, + 0.9147300167236504 + ], + "mean_band_energy": [ + 1.5596861119411924, + 1.4698701819834084, + 1.5290441661809222, + 1.595722664133814, + 2.297336622438481, + 1.5235080047815246, + 1.4508139826578283, + 1.5646820975953861, + 1.517668673795205, + 2.223589141038928, + 2.51682532456534, + 2.4771859732563386, + 2.495780913182842, + 2.5310519375358798, + 2.5215259085977384, + 2.5553684883763084, + 2.5534489568933663, + 2.4487128280158257, + 2.4798961415818392, + 2.5534329627332406, + 2.501219202245625, + 2.567031647124076, + 2.561602868721413, + 2.56094284922536, + 2.4870748486184024, + 2.578257143520762, + 2.5562762970953985, + 2.5441853944659165, + 2.5899778482357414, + 2.496000756629351, + 2.545956286782492, + 2.513781439894418, + 2.5444549170934545, + 2.6011542817163353, + 2.5138321491592257, + 2.480802957630284, + 2.5380376605152803, + 2.487586772745959, + 2.5309073093513117, + 2.4517217926352934 + ], + "dft_magnitude": [ + 92.5159555046915, + 7.997762121204602, + 5.351035682624363, + 2.600294481787762, + 0.311386973452006, + 1.9753827572951956, + 2.4963744344633985, + 2.760987607140328, + 1.5158129658093493, + 0.47100636522491973, + 0.3906035563223464, + 0.30988595295365823, + 0.16542787884356916, + 0.7661361538158029, + 1.4528995812794687, + 1.5666972939753676, + 1.0974767771448715, + 1.0161181653394287, + 0.4212650824850043, + 0.27581074950191914, + 0.06677687461468196 + ], + "mean_band_dot": [ + -3.594532211081966e-06, + 1.2048924509144853e-05, + 2.2217109403754877e-06, + 2.129298849240512e-06, + 9.71380500232044e-06, + -1.7752865801412559e-06, + 1.5872554973839216e-05, + -4.176686047685498e-07, + -6.251593356410013e-06, + 8.646813212465078e-06, + 1.8858127106113898e-05, + 1.5764415525154615e-05, + 1.5054037373829488e-05, + 2.1157116270842374e-05, + 1.9305124070001515e-05, + 1.7107248959291605e-05, + 1.7354665544644373e-05, + 1.4680506444619824e-05, + 2.1055751949461415e-05, + 1.6282422399171993e-05, + 1.6462392136418202e-05, + 1.4833825377991162e-05, + 1.7693377145633345e-05, + 2.2370161968865435e-05, + 2.3098741797866752e-05, + 2.2244784179292762e-05, + 1.822759433594001e-05, + 1.349313141645325e-05, + 1.4837696417657753e-05, + 2.2743594820440186e-05, + 1.4872558872980335e-05, + 2.1831616413692247e-05, + 1.9890603869043844e-05, + 2.1287021274929428e-05, + 1.1778500777381852e-05, + 1.4776789555526193e-05, + 1.6298638365697118e-05, + 1.632966218494403e-05, + 1.8883380784018302e-05, + 1.416734946815268e-05 + ] + }, + { + "layer": 4, + "mean_snr": 0.9086884436854598, + "per_head_snr": [ + 0.8969859077911898, + 0.9002614702627011, + 0.8930642012056417, + 0.9015277515282399, + 0.8940813236345698, + 0.8700692511673853, + 0.878010788600523, + 0.8725631083601765, + 0.8919340817503045, + 0.9273556318913411, + 0.8883275641714826, + 0.9012366601280094, + 0.8361375244250987, + 0.909392573562453, + 0.865825727583413, + 0.9802038717850614, + 0.9570584427511739, + 0.9039434645960177, + 0.88609326161654, + 0.8848906092211856, + 0.9251294887900933, + 0.9865953613504433, + 0.9340723254093173, + 0.9191963540229537, + 0.8816873313538356, + 0.9602683028654591, + 0.9240427638459102, + 0.8850736750232676, + 0.9257295183887503, + 0.9082941883871509, + 0.9643912532207392, + 0.9245864192442839 + ], + "mean_band_energy": [ + 1.4061146875781285, + 1.3155496699247224, + 1.3011545521710526, + 1.4788646286423888, + 1.9799011931322061, + 1.3776692223947502, + 1.348971680164528, + 1.345698739697991, + 1.5236862419294077, + 1.9535969754680278, + 2.1085612505271563, + 2.0681949758109455, + 2.0667191333910275, + 2.119986726744082, + 2.108793731463999, + 2.111794243418217, + 2.106329952810468, + 2.1152619955147567, + 2.044065709604796, + 2.072792731514211, + 2.1204633434478044, + 2.127550013516858, + 2.091184212124182, + 2.093962579616879, + 2.040910308026654, + 2.0906570758346357, + 2.103865289677751, + 2.1093464444136605, + 2.0817611207626174, + 2.1005299446026036, + 2.084420355240022, + 2.088684294666521, + 2.072463328131332, + 2.0685205443932646, + 2.100275442343798, + 2.1458115921285783, + 2.1104240855246377, + 2.0465193396688592, + 2.119117727344421, + 2.1139717332086656 + ], + "dft_magnitude": [ + 77.8641468165766, + 5.3195040924660635, + 3.9608844931550853, + 1.9746181131416471, + 0.15625456023958462, + 1.4669200707680037, + 1.7961667230463756, + 2.1255824386002797, + 1.434131123498292, + 0.99877793287659, + 0.3283595752366602, + 0.33429031734359305, + 0.07808319146620907, + 0.5931925147069571, + 0.7474865190504044, + 0.962757707300629, + 0.8273623075937857, + 0.7088691804259961, + 0.47036016898797883, + 0.34015948609824215, + 0.025780125784635288 + ], + "mean_band_dot": [ + 6.8120887476652575e-06, + 1.9741332528155906e-05, + 9.026266386058524e-06, + -2.8828633276134492e-06, + 2.9598001044206507e-05, + -1.1301590575385467e-06, + 1.5048277498408425e-05, + 8.349252391326447e-06, + 1.3662377909895456e-06, + 3.141830720210237e-05, + 3.9536138791618215e-05, + 4.1586791235204144e-05, + 3.6016907574776266e-05, + 4.373191675313137e-05, + 4.4524641020871064e-05, + 4.021623680614538e-05, + 3.966092426708202e-05, + 4.403385269142745e-05, + 3.706223611743554e-05, + 3.863221591444698e-05, + 3.92981206971399e-05, + 4.362010949989781e-05, + 3.768644189676707e-05, + 3.826810502403077e-05, + 3.135156643452319e-05, + 4.3600345262007063e-05, + 4.2459642980929866e-05, + 4.47968722681935e-05, + 4.4185902231674846e-05, + 4.115845526087015e-05, + 3.845308112602197e-05, + 4.166533204141844e-05, + 3.8704149818613584e-05, + 3.454395731523618e-05, + 4.55345116222361e-05, + 4.561684612554018e-05, + 4.0585720057606516e-05, + 3.134335231038676e-05, + 4.257940396428239e-05, + 3.9201002323352434e-05 + ] + }, + { + "layer": 5, + "mean_snr": 0.8963324336559934, + "per_head_snr": [ + 0.8962837706521849, + 0.8590000079478463, + 0.9274655828636114, + 0.8692533499353418, + 0.9012621027117885, + 0.7724804404664424, + 0.9306559165697298, + 0.9485337191504454, + 0.8869166695740995, + 0.9647578780986424, + 0.8135177129744393, + 0.959491584976832, + 0.9208139735104842, + 0.9218862909368033, + 0.8886020709708454, + 0.9010064398718872, + 0.896655231869015, + 0.8785805570634462, + 0.9193132425088821, + 0.9117144951381885, + 0.8685400406502691, + 0.9267718166714752, + 0.9076458673092352, + 0.8521020993695404, + 0.9284990972481361, + 0.8550168746496558, + 0.9034389789396857, + 0.828858654428941, + 0.9095871264252611, + 0.8865840738983712, + 0.9273511457118454, + 0.9200510638984145 + ], + "mean_band_energy": [ + 1.341128784572461, + 1.428022639118158, + 1.5873446947711822, + 1.8595154883716754, + 2.2872876059821805, + 1.3501129546586434, + 1.4061039919900948, + 1.5552173737382502, + 1.814643412040862, + 2.3117413069236585, + 2.526130825183737, + 2.514308296972082, + 2.5297344621393583, + 2.489024055415282, + 2.500923822933951, + 2.441731387296758, + 2.4547357266955996, + 2.537494841616296, + 2.5309417916761534, + 2.528634156183369, + 2.5260509442071974, + 2.5262349144684504, + 2.4876235102418125, + 2.51069512879232, + 2.5287097075511755, + 2.45963335849203, + 2.479629552739239, + 2.530685267392093, + 2.513199242207215, + 2.527834528172341, + 2.516177318824518, + 2.520988336262405, + 2.421269157329467, + 2.5343309503123606, + 2.5077276718576154, + 2.4721542151396854, + 2.51656222458645, + 2.5231435187768065, + 2.5270999786381405, + 2.495115351004498 + ], + "dft_magnitude": [ + 92.11964249527557, + 7.361550375269152, + 5.24840220344273, + 2.6569145085054444, + 0.2894732082804744, + 2.214392104540457, + 2.940878496902179, + 2.5848697279601183, + 2.2097652640207865, + 1.2679367555042105, + 0.753910238730893, + 0.4664013039946219, + 0.28327505836960315, + 0.4204684111094076, + 1.0790822071557575, + 1.440177968375901, + 1.4079195260605684, + 0.857191104646696, + 0.7418200761382819, + 0.5289690309241596, + 0.11359364293875274 + ], + "mean_band_dot": [ + -2.299287415894469e-05, + 6.27972938502808e-06, + -7.278012782307995e-06, + -4.548125599512785e-06, + 8.390031098315374e-07, + -2.096284512163038e-05, + 5.801149461603929e-06, + -8.824800130469156e-06, + -2.1561206381193195e-06, + 9.998148340173427e-06, + 1.375315833911372e-05, + 1.1152638535349979e-05, + 1.29930387913646e-05, + 1.4972476830621415e-05, + 1.6547996767712903e-05, + 1.8914391375801653e-06, + 1.0741798439539708e-05, + 1.854825312364028e-05, + 1.704397311215189e-05, + 1.867524428575962e-05, + 1.3495077615743868e-05, + 2.0927419080862818e-05, + 1.9959945976211196e-05, + 9.486723655527387e-06, + 1.5085836173511778e-05, + 1.2702857480917373e-05, + 1.5537785463948234e-05, + 9.316395340874805e-06, + 7.424425081126175e-06, + 9.865217521110029e-06, + 8.34774698432739e-06, + 1.760970459372402e-05, + 5.9034750179876055e-06, + 1.2521591543190883e-05, + 1.0233703960693676e-05, + 1.3606689481093783e-05, + 1.0689062685287352e-05, + 1.929188678673199e-05, + 1.3861633681244711e-05, + 1.2190397626454801e-05 + ] + }, + { + "layer": 6, + "mean_snr": 0.891573075148808, + "per_head_snr": [ + 0.9316132260697493, + 0.8869525685641794, + 0.8240266782008392, + 0.8983842766750147, + 0.8752375134331577, + 0.8873378423331287, + 0.8586683801947964, + 0.8986590407241362, + 0.8431654151173322, + 0.8963737179853448, + 0.9316731590356276, + 0.8493596833550665, + 0.912092841097001, + 0.9376399419539695, + 0.9164419135789974, + 0.9017161689919588, + 0.9146774087256068, + 0.9535587461076234, + 0.8897419945018326, + 0.8763715965108146, + 0.9092337657788838, + 0.9660879907286364, + 0.8652698616034782, + 0.953976657185751, + 0.8763242513303303, + 0.8311130463066163, + 0.8364689634240714, + 0.8536482875099637, + 0.9150856197987319, + 0.9157399195743494, + 0.8159485496995514, + 0.9077493786653136 + ], + "mean_band_energy": [ + 1.3476572194412477, + 1.3878064045616068, + 1.4868182606596905, + 1.8848235458976619, + 2.4490822311309297, + 1.366475038145156, + 1.3972160181012299, + 1.5625003533433277, + 1.9645646618244756, + 2.404465601201153, + 2.5387353407418662, + 2.6413594966646445, + 2.6220304792400557, + 2.568798467813116, + 2.605438442071205, + 2.6164368595684087, + 2.6266724593485167, + 2.588387149591378, + 2.617906881810921, + 2.5800352655630276, + 2.5493848595117177, + 2.5799738229152904, + 2.5942542656829777, + 2.6156850840166945, + 2.631918230484847, + 2.569416981540752, + 2.6118867260007868, + 2.613281727331877, + 2.597139471164788, + 2.5833107136863074, + 2.606866539476245, + 2.6198966092743174, + 2.6102990348352915, + 2.612393361447467, + 2.655006628281131, + 2.573925535401482, + 2.5554414998323622, + 2.5486678849703086, + 2.6056724058902896, + 2.5238397497869673 + ], + "dft_magnitude": [ + 95.11547130825151, + 8.040990476963536, + 5.839170185361369, + 2.7201621229291333, + 0.2339758431409332, + 1.925248307596187, + 3.1511074881376095, + 3.082907444575467, + 2.4928132762168302, + 1.6319899829841344, + 1.0486198313734565, + 0.188288636895672, + 0.3494198069846963, + 0.5381227331885418, + 1.0841007245875531, + 1.28413894865295, + 1.5411523668417455, + 1.165456130957565, + 0.7684583887564962, + 0.36342454237312244, + 0.23251200280962792 + ], + "mean_band_dot": [ + -2.6207467966798963e-05, + -1.8491615549010032e-06, + 5.300444155409422e-06, + 1.574649517124271e-05, + 4.4264569044116794e-05, + -2.900936915466445e-05, + 4.860606999557152e-06, + 6.206786849816127e-06, + 1.6827476065373046e-05, + 4.6236474986471876e-05, + 5.820198841774981e-05, + 7.452638176346228e-05, + 6.385509244353216e-05, + 6.358375724744292e-05, + 6.662518894131608e-05, + 6.28143811411519e-05, + 6.517993263059907e-05, + 6.68758147128301e-05, + 6.518512319644288e-05, + 6.416666793711555e-05, + 6.476520723026625e-05, + 6.80399464272341e-05, + 6.842960672202024e-05, + 6.795373832346742e-05, + 6.276911269651463e-05, + 5.862839799988252e-05, + 6.659812463567503e-05, + 6.36377321825421e-05, + 6.265191699412753e-05, + 7.121144160464611e-05, + 6.307751708618526e-05, + 6.422347870511657e-05, + 6.760692391480916e-05, + 6.439750791287224e-05, + 6.632214231103717e-05, + 6.323452607830403e-05, + 6.1016778818157053e-05, + 6.258563697656425e-05, + 6.608744922687038e-05, + 4.7345080474769936e-05 + ] + }, + { + "layer": 7, + "mean_snr": 0.8989221918804801, + "per_head_snr": [ + 0.9599524135535649, + 0.9143858259694388, + 0.8773204243091084, + 0.9087946957750755, + 0.8986961631009768, + 0.9288101529424325, + 0.8334819068835547, + 0.9048667680168659, + 0.8619792326367977, + 0.8584222286086456, + 0.9394878390877475, + 0.8877553726599434, + 0.9106814421642218, + 0.9257862512582063, + 0.9307555038492777, + 0.9329573522021247, + 0.9303648727232314, + 0.8546130634514166, + 0.8111660320379785, + 0.9311203977646296, + 0.8401777425494716, + 0.9082445880903679, + 0.8855718623256695, + 0.929685287441088, + 0.9132235590112352, + 0.9372124665225507, + 0.9062881066055847, + 0.8923274931606694, + 0.9091577209011648, + 0.8846449359256158, + 0.8910679627459455, + 0.8665104759007609 + ], + "mean_band_energy": [ + 1.3672523526417772, + 1.4480953991082757, + 1.5580684632382962, + 1.8550483536685987, + 2.1768234106367794, + 1.3215786425008673, + 1.448325042218242, + 1.5412772439483071, + 1.7923378147504394, + 2.1942477886095664, + 2.3456616157138592, + 2.3799009299087612, + 2.380016975430402, + 2.351160609705209, + 2.3791609578127435, + 2.425920732308389, + 2.3582582279293374, + 2.3617412419750115, + 2.3664043873626683, + 2.419969927495584, + 2.331304355695666, + 2.361944004437932, + 2.3948963487941435, + 2.3848246833615065, + 2.3842512967993628, + 2.425304073342416, + 2.314737605882627, + 2.382509575511835, + 2.342484365714291, + 2.3617182008821347, + 2.3608545996908545, + 2.362382288581516, + 2.351209163726485, + 2.402333756441176, + 2.37633474734809, + 2.356258720461596, + 2.4000272550593773, + 2.4232817716061454, + 2.3968240613416145, + 2.3983174378140517 + ], + "dft_magnitude": [ + 87.98304842945595, + 6.315482395381354, + 4.664040589580355, + 2.4324597749505523, + 0.10312804450029835, + 1.578780122645177, + 2.536956097983771, + 2.3795750074011908, + 1.9146088039717188, + 1.0709497618162107, + 0.856922470741041, + 0.4379846362155494, + 0.08624226513542005, + 0.2426146410549328, + 0.9291557386090026, + 1.2085153158597337, + 1.1955333200359997, + 1.0439733165976253, + 0.7165439980091888, + 0.45833497989197525, + 0.3325823338818239 + ], + "mean_band_dot": [ + 7.540005532291616e-06, + 2.2071146375424174e-05, + 1.9406407830047098e-05, + 1.73214385256415e-05, + 1.5940562661853622e-05, + 3.29311396853882e-06, + 2.017320535969702e-05, + 2.3264462140559775e-05, + 1.6608197384471165e-05, + 1.380980035605717e-05, + 3.3138792787212885e-05, + 2.967691117135018e-05, + 2.9186950212078958e-05, + 3.161254780366107e-05, + 3.490310855340794e-05, + 3.638177629454732e-05, + 3.155660393758808e-05, + 3.070942434533207e-05, + 3.0124226532279864e-05, + 3.075660678177882e-05, + 2.463938000119015e-05, + 3.13699533009526e-05, + 3.104264936837354e-05, + 3.2012872395625884e-05, + 2.808517734820271e-05, + 3.7545454779319686e-05, + 3.3639879330849e-05, + 3.4082880051755635e-05, + 3.3038405354091084e-05, + 3.439257471882228e-05, + 2.835113999708483e-05, + 3.263751329996011e-05, + 3.987739180502103e-05, + 3.184202847705819e-05, + 2.7726226676350052e-05, + 2.628114358458333e-05, + 3.146555027342402e-05, + 3.636661595578517e-05, + 2.9806267025378472e-05, + 3.124569990973214e-05 + ] + }, + { + "layer": 8, + "mean_snr": 0.9160213598809545, + "per_head_snr": [ + 1.088034844262207, + 0.873057645221295, + 0.8540355813092708, + 0.9050978456262838, + 0.920173628352433, + 0.8906323787214212, + 0.8813893290923468, + 0.8432250153221155, + 0.9023042838747755, + 0.9080604405438251, + 0.9448289454327579, + 0.9135953466709147, + 0.9588546308584001, + 0.9478202789736698, + 0.9799839769760759, + 0.9004370101052526, + 0.9067016082023983, + 0.9362723741686928, + 0.8344714695761373, + 0.918218602678389, + 1.0539492020339087, + 0.8870347939336412, + 0.8794595221754927, + 0.9586173143217935, + 0.9283320845860756, + 0.9051025670375716, + 0.9051379746396908, + 0.94646032626178, + 0.9137027544564947, + 0.8904032067616546, + 0.8334971600918825, + 0.9037913739218936 + ], + "mean_band_energy": [ + 1.344176014609504, + 1.4043167749463201, + 1.6031284901185885, + 1.9498772149370538, + 2.1602191047937076, + 1.3312311860542712, + 1.405691997272714, + 1.6151586661947313, + 2.0045129031742883, + 2.1893617876037226, + 2.3599691790898785, + 2.373489139367401, + 2.3787623273159064, + 2.361907946985263, + 2.2887377246890312, + 2.3358947812004027, + 2.3536640772946464, + 2.369447102934107, + 2.377442227056274, + 2.321580378571237, + 2.314426981483728, + 2.357611106394944, + 2.3237657898444275, + 2.327695082356741, + 2.3921312570712696, + 2.3533445421080326, + 2.35386126366398, + 2.349236680218575, + 2.3130145650059633, + 2.342419168795577, + 2.290641403026133, + 2.366317742721135, + 2.3156725982949076, + 2.331190498664714, + 2.333068205182004, + 2.347842780620705, + 2.341620106690208, + 2.375224961114662, + 2.3111402460426356, + 2.3608895431507726 + ], + "dft_magnitude": [ + 87.32968354666016, + 5.7999129933336455, + 4.259583305240214, + 2.339186058988043, + 0.1132554432031055, + 1.5271781957389972, + 2.5254203031839073, + 2.3309326754275252, + 1.99085832573593, + 1.6550205736692303, + 0.9704613432358495, + 0.5258222759170065, + 0.09015477810003757, + 0.27919961893162315, + 0.7661658040793069, + 0.8158276494916035, + 1.03189141545839, + 0.9351787755069433, + 0.8365045625982804, + 0.5850456227065882, + 0.19839062322057543 + ], + "mean_band_dot": [ + -1.970492072587148e-05, + 7.874238744420837e-06, + 3.362331540301966e-06, + -3.4893721817752526e-06, + 2.4286113011839915e-05, + -1.2826023430534404e-05, + 1.2777061630231402e-05, + 1.1641031321119053e-06, + 1.4370468846891526e-06, + 2.6320409597246903e-05, + 4.377218742774858e-05, + 4.194041225673573e-05, + 4.62638227475054e-05, + 4.0049194001312575e-05, + 3.965986211369455e-05, + 4.39191752775514e-05, + 4.018378199361905e-05, + 4.1775925437548275e-05, + 4.232827603232181e-05, + 4.208876405158435e-05, + 3.805806929904065e-05, + 4.586809814099978e-05, + 4.000948261477788e-05, + 4.13888871207746e-05, + 4.444425075291747e-05, + 4.380994944312988e-05, + 3.744840873878274e-05, + 3.4140407160521134e-05, + 3.552467828740191e-05, + 3.548773988768517e-05, + 3.063699548420118e-05, + 4.1313439169243786e-05, + 3.6257588999433206e-05, + 3.689845641474676e-05, + 2.9084090397191175e-05, + 4.388320407429091e-05, + 3.847463005968166e-05, + 4.043940332394414e-05, + 3.3856880509119964e-05, + 3.980402138381578e-05 + ] + }, + { + "layer": 9, + "mean_snr": 0.9075812581738343, + "per_head_snr": [ + 0.8825030075111083, + 0.9163648495585048, + 0.8634439802537488, + 0.8804245528812706, + 0.9165115583977296, + 0.9351327430048244, + 0.9654788056792896, + 0.9141621001138298, + 0.8446074597433035, + 0.8945842427615206, + 0.9507581553722252, + 0.9061158592532718, + 0.912987988142053, + 0.9311324844252046, + 0.8471313788152732, + 0.8705893555106555, + 0.916079217256827, + 0.9045370265661961, + 0.9102659683437678, + 0.9567897869580716, + 0.9351433312033919, + 0.926164688916088, + 0.9496862849164314, + 0.9047726983085, + 0.9834422256822897, + 0.8425045644601418, + 0.8632183012206488, + 0.9624080582855039, + 0.8640369540654476, + 0.8737938197306372, + 0.9136633776441312, + 0.9041654365808112 + ], + "mean_band_energy": [ + 1.315168335378747, + 1.4787980469479953, + 1.6903407013519365, + 2.0139602801785914, + 2.300419416270633, + 1.2721523006438937, + 1.480540883927161, + 1.686026123844876, + 2.011942455356592, + 2.3185851716440253, + 2.4550614858660116, + 2.462449073492848, + 2.446592872784254, + 2.4316255679051797, + 2.4394643310905932, + 2.4414034593957026, + 2.4698237284501205, + 2.4917357581522572, + 2.4526209007769313, + 2.452461799735139, + 2.44119988500977, + 2.478518542988071, + 2.460500729157651, + 2.435097064884321, + 2.4620509194349647, + 2.422594553710299, + 2.4672231920266516, + 2.414765326159982, + 2.4710180521423237, + 2.448354555456885, + 2.444586883268547, + 2.478749676556726, + 2.44510462936497, + 2.4469944200517713, + 2.4835361309205157, + 2.44520825621712, + 2.4655557765653184, + 2.419578098832929, + 2.427393070387553, + 2.4234840571195186 + ], + "dft_magnitude": [ + 91.09268651344938, + 6.377238148351611, + 4.64567206544914, + 2.1880264186174263, + 0.07867029686032098, + 1.6844613005403386, + 2.62746863058066, + 2.5827130347637075, + 2.1092779293791017, + 1.7045884646431504, + 1.0450328369454343, + 0.6369864642367697, + 0.06523029515019252, + 0.509497498760211, + 0.8935196324106149, + 1.2682759661919354, + 1.4623371294932568, + 1.0053816254536863, + 0.9524061401302272, + 0.36777637744669434, + 0.16760224561311077 + ], + "mean_band_dot": [ + -3.365278931255489e-05, + 4.615842973976215e-06, + 9.28765790035868e-06, + 6.185280938098e-06, + 2.127099055542203e-05, + -3.126252419747289e-05, + 1.480648566598573e-06, + 1.3390518955702645e-05, + 5.946067437889725e-06, + 1.773931820423513e-05, + 3.705428430293977e-05, + 4.340826571933576e-05, + 3.339180061630031e-05, + 3.390021815903309e-05, + 3.677028964830243e-05, + 3.6758210910647904e-05, + 3.6138106770522415e-05, + 4.071100663054495e-05, + 3.539924771303049e-05, + 3.4939228430630455e-05, + 3.766310586001965e-05, + 3.708228384766698e-05, + 4.2675912874301494e-05, + 3.2687623422589235e-05, + 4.319411984283761e-05, + 3.5570108821048045e-05, + 3.772379407678272e-05, + 2.9397600235370196e-05, + 3.3322901060728334e-05, + 3.2436166287652667e-05, + 4.0851582116374634e-05, + 4.119395807720138e-05, + 3.78791918706156e-05, + 3.7103035344898666e-05, + 3.4519974019531215e-05, + 3.093305932679868e-05, + 3.436456026975065e-05, + 3.4637794215086615e-05, + 3.205515051831752e-05, + 3.616312589826976e-05 + ] + }, + { + "layer": 10, + "mean_snr": 0.9091512162152712, + "per_head_snr": [ + 0.9307227558877834, + 0.9475072369951293, + 0.8874621606673505, + 0.9362327841207807, + 0.9145847248062782, + 0.8995044129241081, + 0.9207404382164632, + 0.880668579894653, + 0.848432329456032, + 0.8651640615233849, + 0.9514802044266184, + 0.8793523904869207, + 0.8781953712213398, + 0.8998211002199165, + 0.9527548579573539, + 0.8627476197622871, + 0.8992309348839207, + 0.8464644127439209, + 0.8343863270393357, + 0.9284553537603211, + 0.9880020536865397, + 0.918476575846216, + 0.965470480131691, + 0.9031150695594331, + 0.9203526405785911, + 0.8173009767243745, + 0.9516509208762479, + 0.9665645774401863, + 0.8825026902934766, + 0.9399260680245178, + 0.9048027429202151, + 0.97076606581329 + ], + "mean_band_energy": [ + 1.3214386295408986, + 1.4802330560541384, + 1.7149559583343197, + 2.1434985589249305, + 2.3293090396611427, + 1.3130741385992537, + 1.4165993723337995, + 1.6187004013659345, + 2.197452841038987, + 2.4089385278454056, + 2.5242739803377137, + 2.527679549964777, + 2.513838705182304, + 2.4659280398221664, + 2.5169353938900114, + 2.5176630204943615, + 2.5400228134156575, + 2.5034833133909165, + 2.5408851624163638, + 2.441596001289458, + 2.505365909292881, + 2.560728215511786, + 2.5044867587755397, + 2.526085120659551, + 2.5154950991204825, + 2.5335484263798853, + 2.5333976880197357, + 2.5220258954917902, + 2.525419170233354, + 2.513349512498965, + 2.509983732415823, + 2.549912531214082, + 2.529479389237887, + 2.4935799180759926, + 2.504355459003429, + 2.5026219749283936, + 2.5029887434920215, + 2.534717799171739, + 2.5233567379686175, + 2.5531028961725917 + ], + "dft_magnitude": [ + 93.48050748156709, + 6.591988914640914, + 4.754408039192027, + 2.534555584880233, + 0.2133073416963958, + 1.640142388370255, + 2.9789766405392086, + 3.225286146336238, + 2.4681321874596187, + 1.9710778973095178, + 1.1119334437802844, + 0.5716535065262638, + 0.2805124919256464, + 0.4329802786948471, + 0.70929597165157, + 1.0344287223335038, + 1.2319671685629745, + 1.2820525272226782, + 0.9546831374748469, + 0.7320752083102572, + 0.05957368585485057 + ], + "mean_band_dot": [ + -2.2839755388304184e-05, + 2.674503687671857e-06, + -1.0372546796588729e-06, + 6.654915546278062e-06, + 3.165540468899054e-05, + -2.6715478645655823e-05, + 6.191280004941292e-06, + 7.870878494031786e-06, + 9.522362620373317e-06, + 4.265752131971112e-05, + 5.5930471535248214e-05, + 5.699725901990861e-05, + 5.511165475127201e-05, + 5.399166258257538e-05, + 5.9067589006644994e-05, + 5.677627925706474e-05, + 5.918867238676739e-05, + 5.952789296372885e-05, + 6.231493382529152e-05, + 5.28061044150263e-05, + 5.897898328157678e-05, + 6.0983770242728494e-05, + 5.66615696016015e-05, + 5.892338057833512e-05, + 5.421253254098701e-05, + 6.300356085375824e-05, + 5.954131440262244e-05, + 6.28952840486363e-05, + 6.424083171907569e-05, + 5.887238344257638e-05, + 5.741575494084828e-05, + 6.0202806548659265e-05, + 6.335487852595636e-05, + 5.988842311808185e-05, + 6.538330403600414e-05, + 6.33031954521357e-05, + 5.689625859872648e-05, + 6.0489379956152326e-05, + 5.44806330992742e-05, + 6.160338449490155e-05 + ] + }, + { + "layer": 11, + "mean_snr": 0.8972664630416485, + "per_head_snr": [ + 0.8362224423597535, + 0.9242452706133035, + 0.9365769426055395, + 0.9914497982592793, + 0.8176144784324108, + 0.901496784488891, + 0.8770258085065125, + 0.9436818433947253, + 0.9360952720503048, + 0.9330492551006139, + 0.9048669924200641, + 0.8988609335362473, + 0.9392551672090073, + 0.8462468423232401, + 0.8821345737188632, + 0.8517049487557361, + 0.9542460113002174, + 0.9280731586625297, + 0.8288371533714953, + 0.8844051384756575, + 0.9003412885188147, + 0.8634151698122163, + 0.8707282083443783, + 0.8853527758271096, + 0.8999023951116614, + 0.8478591919221761, + 0.9322404031345457, + 0.9224970463260925, + 0.811270186619497, + 0.9861461189558486, + 0.8529482161742958, + 0.9237370010017277 + ], + "mean_band_energy": [ + 1.2122108262677445, + 1.459040425161207, + 1.722187514255646, + 2.3025252450703313, + 2.633070421423995, + 1.182162389423614, + 1.4545279274483558, + 1.7202338632793752, + 2.2914501773127487, + 2.6180816486762843, + 2.7073028215907677, + 2.7200546501126337, + 2.6963054591652043, + 2.7365502814793405, + 2.734318498868628, + 2.7035836985602373, + 2.739230021586295, + 2.7196614609897853, + 2.711021803440397, + 2.72448486188021, + 2.744624558996703, + 2.6592981804245923, + 2.734292790181378, + 2.6864588506136586, + 2.723765020593254, + 2.73098149573184, + 2.7266140607009515, + 2.7357252351335934, + 2.6816624517483247, + 2.7254295187482787, + 2.7125977546601487, + 2.7253369926388746, + 2.690222035505709, + 2.7307873300250067, + 2.7336619816214247, + 2.73507827550338, + 2.7126539463977117, + 2.7059192917177057, + 2.7023880875198283, + 2.706837637809915 + ], + "dft_magnitude": [ + 100.09233949226508, + 7.820161171650641, + 5.772277893724466, + 2.8269827816303823, + 0.10664068354173799, + 2.303026359313702, + 3.4903178286843843, + 3.784940131486972, + 3.2958043942112805, + 2.47466800639562, + 1.5039881574563176, + 0.6231754982552508, + 0.15388269109464756, + 0.7250611704972549, + 1.1364609670611385, + 1.5598862185513225, + 1.7750420672927358, + 1.708105512835879, + 1.1843035327652802, + 0.8174147278585177, + 0.035876826305354825 + ], + "mean_band_dot": [ + -2.472017999934906e-05, + -7.312495836231392e-06, + 9.04330793787267e-06, + 2.28492210112563e-05, + 3.95768007138031e-05, + -2.5958063645248327e-05, + -5.370535495785589e-06, + 4.129560812771161e-06, + 1.8733385593350247e-05, + 3.9067722752861293e-05, + 5.545082597109286e-05, + 6.693102226336123e-05, + 5.922144209762337e-05, + 5.664716404680803e-05, + 5.456202790696806e-05, + 5.696068872111936e-05, + 5.400413017184747e-05, + 5.312655298439496e-05, + 5.25385214700691e-05, + 6.377177828653657e-05, + 5.802155424419198e-05, + 5.554340951050564e-05, + 5.0116037914449405e-05, + 5.341322366803068e-05, + 6.017872052552776e-05, + 5.9307873860348096e-05, + 5.796371217456908e-05, + 5.903475525315117e-05, + 4.614462488916616e-05, + 5.641539660530271e-05, + 5.8239736560494735e-05, + 5.5338953131922615e-05, + 5.381996389530741e-05, + 6.0943594780837886e-05, + 5.589847683040717e-05, + 5.501837773635999e-05, + 5.5700626501220536e-05, + 3.968640546077039e-05, + 5.317547893923801e-05, + 5.603773431630544e-05 + ] + }, + { + "layer": 12, + "mean_snr": 0.9037374041749453, + "per_head_snr": [ + 0.8631008514414349, + 0.9223807028802037, + 0.8957524262820239, + 0.909307096348555, + 0.8510511521156348, + 0.9669516419031299, + 0.9301195584559906, + 0.8255694819296991, + 0.9108369495210538, + 0.8812423932683168, + 0.9039870920330373, + 0.8956953491039873, + 0.9383065843755023, + 0.8859449368943059, + 0.9694293024044179, + 0.9427454209698564, + 0.8434450156551787, + 0.9189695145584179, + 0.878747120649408, + 0.9167482787852105, + 0.8992862948547201, + 0.9193189960096931, + 0.9105570559567479, + 0.88099927257232, + 0.8263485536245468, + 0.9102934217075749, + 0.9135295359148993, + 0.9346856231696129, + 0.8653262062896979, + 0.9308946837913402, + 0.8748748610935435, + 1.0031515590381863 + ], + "mean_band_energy": [ + 1.30444801701731, + 1.5750243068684593, + 1.7739431303939972, + 2.1637563023067248, + 2.4818710357769254, + 1.2774635560287142, + 1.5190499401071171, + 1.7297126467000812, + 2.115634961422233, + 2.4779282823748288, + 2.6150186049976787, + 2.5724510716331945, + 2.5997500257187984, + 2.563008572003195, + 2.591832219143276, + 2.5811938592071, + 2.600972451400191, + 2.572005308119264, + 2.5807296595468756, + 2.6174890063487903, + 2.6000075383830277, + 2.608503473016389, + 2.5981302619107005, + 2.5463045748845325, + 2.5833975919402725, + 2.588815546753014, + 2.5969674939168335, + 2.616084661095701, + 2.626031051824765, + 2.5925142177108156, + 2.613437349863044, + 2.6031647112909697, + 2.581371120139047, + 2.619182856943336, + 2.593395755401328, + 2.6209655380417884, + 2.5812041509143824, + 2.5552781979329127, + 2.607742527047497, + 2.598015449009645 + ], + "dft_magnitude": [ + 96.24379702513475, + 6.859266902437787, + 4.965912737912759, + 2.362065011653003, + 0.18723234099973288, + 2.062888465502946, + 2.8090139594702515, + 3.1364069049311527, + 2.516266289639302, + 1.8857549966229519, + 1.2195127605499725, + 0.6099086999138552, + 0.0607825574179747, + 0.5920565146257367, + 1.2968121575333018, + 1.5261805171598386, + 1.541825255489451, + 1.2399725828776833, + 1.09038065507065, + 0.666719152255965, + 0.08607274859583924 + ], + "mean_band_dot": [ + -2.3876420198121195e-05, + -4.79450887951316e-06, + 7.880631613943478e-06, + 2.526454371718501e-05, + 4.158291469593678e-05, + -2.8309701957596192e-05, + -3.904818140654244e-06, + 1.0085695859629596e-05, + 2.2991759647084105e-05, + 4.288150225875142e-05, + 6.0756033280995336e-05, + 5.0853406389705934e-05, + 6.008006712363567e-05, + 5.0401566514324256e-05, + 5.308646971116105e-05, + 5.34415895458551e-05, + 5.650525404234941e-05, + 5.529853478378753e-05, + 5.5520987962154324e-05, + 5.579172278089573e-05, + 5.525272645172664e-05, + 5.905521733211571e-05, + 5.723475284185042e-05, + 5.607614562563867e-05, + 5.281223787960698e-05, + 5.841461610884836e-05, + 5.581438643957882e-05, + 6.047939215818586e-05, + 4.956759796641564e-05, + 5.633134884988068e-05, + 5.396498358720692e-05, + 5.960038154171343e-05, + 5.324809862372603e-05, + 6.0296713174068385e-05, + 5.3737925668428936e-05, + 5.833502211771701e-05, + 5.3466833037418824e-05, + 5.7565292954109276e-05, + 5.875537578958756e-05, + 5.597272404997967e-05 + ] + }, + { + "layer": 13, + "mean_snr": 0.8902613310289421, + "per_head_snr": [ + 0.8752118938994274, + 0.8955021975562357, + 0.8512342765744072, + 0.8704034762865778, + 0.9037385917269386, + 0.9215124681754671, + 0.892666420446439, + 0.9396628051103607, + 0.9458558865484478, + 0.8132273975501251, + 0.9202257885645022, + 0.869410267811729, + 0.8779430977514371, + 0.9309960949267692, + 0.8702453525002631, + 0.939491364580472, + 0.8748194044029289, + 0.9523870158556523, + 0.8906513475748641, + 0.858805661673866, + 0.86492353117477, + 0.953377816340311, + 0.845420421017551, + 0.8288011327973294, + 0.8261449252805675, + 0.9263679617166011, + 0.8993087664685308, + 0.9136285485457687, + 0.832891987656942, + 0.8547680678849693, + 0.8926068946709904, + 0.9561317298549079 + ], + "mean_band_energy": [ + 1.1935384824879205, + 1.5730988788986362, + 1.823763803905605, + 2.325804512921021, + 2.805311293119843, + 1.2182857087354337, + 1.5603213026841465, + 1.8581905628710098, + 2.2852238350300014, + 2.8145511217934702, + 2.8782209590668546, + 2.9039792048881967, + 2.9253715739434383, + 2.931077339563245, + 2.919045144534938, + 2.8940655603816787, + 2.890129625572528, + 2.905360151610787, + 2.9147768936495995, + 2.9060555076560726, + 2.8890555647301337, + 2.9296002934573395, + 2.896574541550201, + 2.9421933441847954, + 2.9017642511827657, + 2.8755078266566936, + 2.922057458477067, + 2.9143272115879992, + 2.921562343442712, + 2.905777303066072, + 2.93625132262959, + 2.897781338371664, + 2.908871533669, + 2.916466430462846, + 2.919278064907817, + 2.909853880347398, + 2.8854035408152057, + 2.919036153873971, + 2.9048183760290978, + 2.918595095772722 + ], + "dft_magnitude": [ + 106.74094733852951, + 8.781259732620782, + 6.401566404782498, + 3.1723051553202586, + 0.07225116072504235, + 2.531625971250187, + 3.741273394227076, + 3.899105201586651, + 3.4568044216698364, + 2.390937984132162, + 1.4836845549114075, + 0.6754180907132183, + 0.09307156105309548, + 0.7850559577006782, + 1.6221680645039145, + 2.0678356352880227, + 2.210608360444413, + 1.908308533364586, + 1.4291706385933425, + 0.6303941018551144, + 0.17826751567258015 + ], + "mean_band_dot": [ + -2.5142622439489058e-05, + -6.586990127743772e-06, + 5.103006367335183e-06, + 2.33105462086769e-05, + 3.836135195456335e-05, + -2.1071810257922157e-05, + -1.086159928718189e-05, + 3.4038776334455192e-06, + 2.3387836512256397e-05, + 4.868167055747108e-05, + 6.336023587891758e-05, + 6.0423138370424575e-05, + 7.003275377428509e-05, + 6.987711913666316e-05, + 6.437636550344905e-05, + 6.454866224885336e-05, + 6.241608432446811e-05, + 7.055853637893961e-05, + 6.564788604919157e-05, + 6.760269288292876e-05, + 5.936286500514143e-05, + 6.546533925302355e-05, + 6.593272635200265e-05, + 6.701684159224896e-05, + 6.569239919826942e-05, + 6.294666483199762e-05, + 6.613978218865667e-05, + 5.8657086810853805e-05, + 6.838855595105998e-05, + 6.279252935996736e-05, + 6.854485260419096e-05, + 6.003185855121274e-05, + 6.27603486350381e-05, + 6.563569392596945e-05, + 6.550151008468674e-05, + 6.301717093037951e-05, + 6.680425331069274e-05, + 6.94565881985909e-05, + 6.450041157677334e-05, + 6.52865554741311e-05 + ] + }, + { + "layer": 14, + "mean_snr": 0.9043661887636482, + "per_head_snr": [ + 0.8623064736141397, + 0.8766966845068936, + 0.8807258421653212, + 0.943839317253292, + 0.9082249981673738, + 1.0005369293035977, + 0.8961574498270876, + 0.9124707599112364, + 0.9470792036456407, + 0.9023438556151334, + 0.8594571957453712, + 0.8994942476620198, + 0.9627109442432416, + 0.8993713136446968, + 0.8817159522013093, + 0.9055939484613125, + 0.8785936862684137, + 0.8716885526724858, + 0.9702810241737355, + 0.8842865135218301, + 0.8765730651156654, + 0.9654981135612686, + 0.8761446838715305, + 0.8243876779690734, + 0.8995349282425903, + 0.8930867983074803, + 0.966995328301209, + 0.9146471478138822, + 0.9194722216132755, + 0.9255371803338385, + 0.8792999567140366, + 0.854966045988759 + ], + "mean_band_energy": [ + 1.397534287294512, + 1.6178092465986926, + 1.7564674338521833, + 2.1752265412311655, + 2.6319301632730814, + 1.3651260861366392, + 1.6588154392418253, + 1.7757982020667418, + 2.1311579791189206, + 2.4683659766695714, + 2.7318108136059616, + 2.742671000595608, + 2.7000204660217877, + 2.674942643166652, + 2.7140870324111326, + 2.6890442655502302, + 2.659572239869725, + 2.7075563082157643, + 2.725685867896462, + 2.742854470429513, + 2.678970336382163, + 2.6867941651303395, + 2.7254527607268897, + 2.724462034223611, + 2.6757690345754663, + 2.6927535264983136, + 2.7050097714342503, + 2.7178854219867317, + 2.7266565033129275, + 2.65259170403696, + 2.7179019938294506, + 2.7100618516199626, + 2.695692114341843, + 2.71738566039479, + 2.710339102740584, + 2.7027833897288893, + 2.694308535789512, + 2.696989580576233, + 2.7007418429513734, + 2.6880138777299907 + ], + "dft_magnitude": [ + 100.08703967125645, + 7.285549889510808, + 5.215715768151007, + 2.519051982968526, + 0.12581169747976914, + 2.2016547710414134, + 3.0460709164287505, + 2.962689133139719, + 2.4401046341284545, + 1.5694826863988798, + 1.3970776727127505, + 0.6394960188979876, + 0.2595947158649644, + 0.7053101148162787, + 1.184255560932574, + 1.6457645980280058, + 1.5588415266316342, + 1.461214798066275, + 1.2972959820483918, + 0.6515106860155297, + 0.1888077660836558 + ], + "mean_band_dot": [ + -7.3753233095885635e-06, + 8.495090870042078e-06, + 1.228872774561296e-05, + 3.0177016117249878e-05, + 5.110663623781874e-05, + -3.4766129630270377e-06, + 1.0570092263151307e-05, + 1.0770154523243036e-05, + 2.566149909739579e-05, + 3.2050148536200144e-05, + 6.366964641415506e-05, + 6.888120300345692e-05, + 6.352661513631119e-05, + 5.305955342009837e-05, + 6.837200683094124e-05, + 6.493491134733631e-05, + 6.891142172662513e-05, + 6.606539151050584e-05, + 6.857570896414699e-05, + 6.129162093202468e-05, + 5.985957877783222e-05, + 6.209693763139512e-05, + 6.425872084037111e-05, + 6.677119725395642e-05, + 5.553236640025717e-05, + 6.548306972433694e-05, + 6.719243885981995e-05, + 6.31032911201146e-05, + 6.30664705226991e-05, + 6.001970970430647e-05, + 6.007346073602092e-05, + 6.421771263376286e-05, + 6.87147882217687e-05, + 6.210822634784563e-05, + 6.456015245248636e-05, + 6.642599972224162e-05, + 5.971880464130662e-05, + 6.838502522441556e-05, + 5.9277609489072346e-05, + 6.191633317484957e-05 + ] + }, + { + "layer": 15, + "mean_snr": 0.8800042840660901, + "per_head_snr": [ + 0.8857839973574781, + 0.8487846566192324, + 0.8793871779742397, + 0.8925555254450692, + 0.8694498723885772, + 0.9173794974359881, + 0.843344967376014, + 0.9037664385582761, + 0.8691042171983111, + 0.8465247176969677, + 0.8734240176592335, + 0.82352914229353, + 0.8771814331831487, + 0.8529541201869374, + 0.900610431787736, + 0.92843140292908, + 0.8695892081595412, + 0.901784436830256, + 0.9075864357619892, + 0.8674684381083919, + 0.8824715748759026, + 0.9108421183593219, + 0.8487230557766601, + 0.8892428775658197, + 0.9095435984828122, + 0.9197038626482534, + 0.8605409298507657, + 0.8802638852869634, + 0.8947997043967889, + 0.8862868309965437, + 0.8455320654627205, + 0.873546451462335 + ], + "mean_band_energy": [ + 1.3993063447230025, + 1.5555835142104053, + 1.7255797349414568, + 2.0303378729776558, + 2.6295946813441775, + 1.3534881191549366, + 1.567001674728773, + 1.7323528107632198, + 1.946071994434811, + 2.6327584129152917, + 2.830424517758156, + 2.8117553304017386, + 2.842020782255271, + 2.793856094516213, + 2.8126280474777947, + 2.7895204961800992, + 2.7688971543349097, + 2.7892668346830507, + 2.8398441477577374, + 2.7598911830036763, + 2.843724981785048, + 2.83569560533394, + 2.8102783314818223, + 2.8058319833806733, + 2.741482499554012, + 2.802587496498895, + 2.8131873476885274, + 2.813233730112904, + 2.807461251403622, + 2.8328226030481964, + 2.827982033144948, + 2.867659269668952, + 2.7651820647771514, + 2.796539459323248, + 2.7948729145374474, + 2.829314532070715, + 2.832708594662826, + 2.806578831128289, + 2.8500833230397573, + 2.8331391708783666 + ], + "dft_magnitude": [ + 102.92054577208172, + 8.545463784087106, + 6.304607349444567, + 3.139049672663157, + 0.26908151611448483, + 2.5326536962104074, + 3.390427033160611, + 3.284834875860695, + 2.3141047485049375, + 1.7118168247885122, + 1.1137566756768342, + 0.7510534112541006, + 0.06183810747016402, + 0.7066264197483322, + 1.4547955250951181, + 2.1495519000436185, + 1.7660537879946765, + 1.4936395917827598, + 0.8448883500539153, + 0.6830352002897827, + 0.023880928419217184 + ], + "mean_band_dot": [ + 1.73721874716648e-06, + 4.809906988612057e-06, + 7.110619253580807e-06, + 3.1399055228575885e-05, + 5.4241928819465104e-05, + 1.051975469579247e-06, + 7.506644422505814e-07, + 7.181986916293681e-06, + 2.5754627279184207e-05, + 5.549669342599373e-05, + 7.799665606853522e-05, + 7.916066715552006e-05, + 7.867501644795995e-05, + 7.874548646213952e-05, + 7.35598878463861e-05, + 7.896221537748713e-05, + 7.463612837455001e-05, + 8.14678669485147e-05, + 7.990683734533377e-05, + 7.749101250738022e-05, + 7.652287813471045e-05, + 7.967879718080439e-05, + 7.721213769400494e-05, + 7.629479941897443e-05, + 7.146383447889094e-05, + 7.351832097128862e-05, + 7.560466408449428e-05, + 7.92985988766759e-05, + 7.040973302707698e-05, + 8.098047744624634e-05, + 8.01418990363345e-05, + 7.202271663118154e-05, + 7.307213345484343e-05, + 7.697152905166148e-05, + 7.65645060823772e-05, + 8.279955472971779e-05, + 7.87761850915558e-05, + 7.708303596700717e-05, + 7.750598963411906e-05, + 7.986309210536998e-05 + ] + }, + { + "layer": 16, + "mean_snr": 0.8910919261304544, + "per_head_snr": [ + 0.8682068779780886, + 0.9211517007689595, + 0.8455748393419453, + 0.9178836002895331, + 0.8597985830125144, + 0.8904679130520471, + 0.9015663336071145, + 0.9291935922745547, + 0.8743112175343793, + 0.9212153922675277, + 0.8982187658837819, + 0.864269347353838, + 0.917396269835874, + 0.8839107273609099, + 0.8598734430547539, + 0.8790878482513471, + 0.8804532612928757, + 0.9136322242345015, + 0.9378008459271958, + 0.890083450712521, + 0.8759537038266096, + 0.9240949789061861, + 0.8701535840346867, + 0.9819641703739959, + 0.8804317851808685, + 0.8609898537995071, + 0.8807911392043751, + 0.9050724664880666, + 0.8483132191489741, + 0.865614601902434, + 0.8808081443202461, + 0.8866577549543279 + ], + "mean_band_energy": [ + 1.4457785171375017, + 1.6294166181307828, + 1.7081005481118603, + 1.9557559747871662, + 2.5378763492331844, + 1.4888389467527603, + 1.5683633879894299, + 1.7380682376884826, + 2.031025658790087, + 2.552119677999479, + 2.8051421908988408, + 2.7867317649292698, + 2.8042763952628325, + 2.7062451651571617, + 2.77672772682766, + 2.768927161824193, + 2.7802753258948436, + 2.749742390448655, + 2.744767793087917, + 2.77958721033787, + 2.7464747921464165, + 2.7451917219033306, + 2.7740886152221234, + 2.775805289047819, + 2.7617406768044526, + 2.796651982635502, + 2.777192380193106, + 2.774137693843187, + 2.726800170532867, + 2.7521353151622745, + 2.769842956418743, + 2.7637308954997892, + 2.74035452711087, + 2.7763577843684573, + 2.746105595417141, + 2.741444824263223, + 2.784679369846881, + 2.754005751226545, + 2.7592672560443945, + 2.7932488377942835 + ], + "dft_magnitude": [ + 101.61702347677138, + 8.108210885388862, + 5.885515498325024, + 3.1332853293936926, + 0.07415360786434133, + 1.965156803673364, + 3.1581041468088342, + 2.9919452229321655, + 2.063107917801648, + 1.3316323649960462, + 0.9293963779945374, + 0.3191962488116591, + 0.09876988994807377, + 0.7363860846455726, + 1.384973097583468, + 1.7425898800154842, + 1.6652819734520727, + 1.228643332073551, + 0.8878938708656274, + 0.6634373466682832, + 0.039263010829074574 + ], + "mean_band_dot": [ + 1.712122361823276e-05, + 2.2455503122387198e-05, + 1.436696663290604e-05, + 3.265561401519789e-05, + 4.8894878727878666e-05, + 1.5269294772224383e-05, + 2.101083974821449e-05, + 1.945483010175053e-05, + 3.1013277342140115e-05, + 4.9687607383930295e-05, + 7.444849941293797e-05, + 7.794327108854306e-05, + 7.127763469725324e-05, + 7.08001004795733e-05, + 7.406244208141288e-05, + 6.994453572133351e-05, + 7.243007215151918e-05, + 7.400100019196997e-05, + 7.079204781348381e-05, + 7.720797170804874e-05, + 6.755925204515732e-05, + 7.629475172734599e-05, + 7.742272304511742e-05, + 6.968963022018216e-05, + 7.276971496139596e-05, + 7.925606774961123e-05, + 7.474316619209277e-05, + 7.464911501529058e-05, + 6.74825025953396e-05, + 7.332033401326043e-05, + 7.278687546659056e-05, + 7.35886154188847e-05, + 7.223064890240493e-05, + 7.804129264741278e-05, + 8.11788931855517e-05, + 7.121442642983311e-05, + 7.70510001530056e-05, + 6.79952740142653e-05, + 7.776143072533157e-05, + 7.194395695648835e-05 + ] + }, + { + "layer": 17, + "mean_snr": 0.8720213938918475, + "per_head_snr": [ + 0.8861624249828445, + 0.8505001029873379, + 0.8780834193922097, + 0.8611036406061875, + 0.8478705067005655, + 0.8696986565304242, + 0.9055944622108877, + 0.9003111403668489, + 0.8245654775680762, + 0.8548546379266554, + 0.8886305109374104, + 0.8493292953788997, + 0.8669884710655023, + 0.8483472423955823, + 0.8469049119866462, + 0.8396235073503932, + 0.9090853921309888, + 0.8646047276318478, + 0.8957669590534104, + 0.9383490294406706, + 0.8656984065548591, + 0.875607934569552, + 0.8464380557459026, + 0.8922307113400572, + 0.8756150849975934, + 0.8905917998186773, + 0.8645982776080381, + 0.882628614284672, + 0.8583238334399794, + 0.9436334887509078, + 0.858027408928483, + 0.8249164718570091 + ], + "mean_band_energy": [ + 1.358983605161741, + 1.5407756019990326, + 1.6986529866763256, + 1.9258831640876009, + 2.746422982923717, + 1.3892161885958223, + 1.5182280266740737, + 1.6881253621421346, + 1.8878247125657408, + 2.6979726919154134, + 2.9717483372813156, + 2.9845125626473044, + 2.9768386842119154, + 2.9456245444593256, + 2.9412836726409464, + 2.989084427162779, + 2.9085648612361403, + 2.9473723089818282, + 2.9985740450658427, + 2.9814406681461096, + 2.9177774815850093, + 2.9392195677497774, + 2.962289275205507, + 2.949418390119074, + 2.9065526437128546, + 2.9809176861478552, + 2.971556441748637, + 2.968106507048963, + 2.9597136785429714, + 2.94520493883084, + 2.958116057454249, + 2.995205361560611, + 2.9331635410576404, + 3.019974005834294, + 2.9584124252678374, + 2.990627368481283, + 2.94902819939587, + 2.8654083891007742, + 2.9741887606421393, + 2.9453800851148566 + ], + "dft_magnitude": [ + 107.18739023917615, + 10.076126247032754, + 7.35621542248797, + 3.4278839053186387, + 0.1728823173162883, + 2.7321610845033737, + 3.7763140630539924, + 3.5835368658866233, + 2.6006417027094364, + 1.5231826329490605, + 1.2166146393038753, + 0.6003291638597456, + 0.24976185257691305, + 0.9190102492378313, + 1.9111348199786808, + 2.42685288001733, + 1.9469038661555682, + 1.5323764354113985, + 1.086531386828382, + 0.5753828822064636, + 0.19154940107520702 + ], + "mean_band_dot": [ + 9.128098656674408e-06, + 7.190948403490436e-06, + 4.184252162531266e-06, + 3.138738140933128e-05, + 8.410418649873462e-05, + 1.317092963262212e-05, + 1.675771557074768e-06, + 6.834538137923117e-06, + 4.074824789768172e-05, + 7.717206904089836e-05, + 8.705414794576427e-05, + 9.222392495757956e-05, + 9.289456215810789e-05, + 9.028663439494266e-05, + 9.510375075478806e-05, + 9.331832422958542e-05, + 9.340599863207896e-05, + 9.080014438040959e-05, + 9.680454395493144e-05, + 9.436537241072075e-05, + 9.023263310155017e-05, + 8.774640482442917e-05, + 9.207706789311487e-05, + 9.644090406482065e-05, + 9.351448250072279e-05, + 0.00010194131273237872, + 9.394106609761367e-05, + 9.414553924216306e-05, + 9.602809006992173e-05, + 9.940443837876954e-05, + 9.54596489918913e-05, + 9.510895454241108e-05, + 8.865935940320924e-05, + 9.468364214626491e-05, + 8.691093992183597e-05, + 9.645293289395339e-05, + 9.800918012388138e-05, + 8.868419783993886e-05, + 9.132539735219323e-05, + 9.093326434594926e-05 + ] + }, + { + "layer": 18, + "mean_snr": 0.8764737314352692, + "per_head_snr": [ + 0.8832930468641746, + 0.9250044766403832, + 0.8870080176285734, + 0.877903327268852, + 0.8533387411959499, + 0.9441716423341251, + 0.9291176068173479, + 0.8776537963905474, + 0.8468057395176235, + 0.9235623035675887, + 0.8491476122311415, + 0.8534364899909991, + 0.8220482249658944, + 0.9113283594195284, + 0.8282589783785352, + 0.8774315499231076, + 0.8350638920386296, + 0.8609425086661987, + 0.8850342709682271, + 0.8804121944226648, + 0.8471565590473745, + 0.8799395147074884, + 0.8747489359796593, + 0.8934180630696766, + 0.9143400073631751, + 0.8819034955541082, + 0.871038328461148, + 0.9300874598417375, + 0.8200785925505903, + 0.8889820714032571, + 0.842434939574505, + 0.852068659145802 + ], + "mean_band_energy": [ + 1.578230876339705, + 1.6681007081166417, + 1.7738054078208343, + 1.8725061980727333, + 2.7747469836824634, + 1.5464543233521293, + 1.6699350950394827, + 1.814123524608374, + 1.97951039890409, + 2.784834715062427, + 3.0558903965371584, + 3.077536878016953, + 3.0731848297236866, + 3.0464336063881126, + 3.0778164934732546, + 3.0909639735445165, + 3.0564288112492326, + 3.1338295011429023, + 3.1398717122539943, + 3.0993029754707098, + 3.171988250945681, + 3.1192881421065612, + 3.129828827234754, + 3.0716847585487477, + 3.0781864414106628, + 3.0721748803797513, + 3.1252277931445755, + 3.0976723002134805, + 3.098863877499591, + 3.05258557232238, + 3.1247170838718876, + 3.1065340737649563, + 3.061426243324172, + 3.0750412967861376, + 3.0447647666439073, + 3.14375897971637, + 3.115273841889899, + 3.100221029976587, + 3.0855499279058147, + 3.076627378014762 + ], + "dft_magnitude": [ + 112.26492287450007, + 10.580699953589656, + 7.434013837478202, + 3.667449050250352, + 0.3801217187199278, + 2.6659269599047586, + 3.5777369351110173, + 3.3149988233369894, + 2.060263712705146, + 1.3176112166722906, + 0.8847969966831599, + 0.5915993466136032, + 0.17648624960423465, + 0.9284310779796903, + 1.7060287434782726, + 2.2957075265371945, + 1.9055903595026584, + 1.6049775075222898, + 1.2336845064944755, + 0.4692117906860554, + 0.16557324328961442 + ], + "mean_band_dot": [ + 3.040486384406904e-05, + 2.0630249267128423e-05, + 1.845407280143263e-05, + 3.70063812852095e-05, + 6.593875058342747e-05, + 2.388323985087482e-05, + 2.4593111248805144e-05, + 2.041345731527145e-05, + 3.626737496773557e-05, + 7.269331826478266e-05, + 7.938228128807624e-05, + 8.596289314937164e-05, + 9.122092876623357e-05, + 8.092646376098853e-05, + 9.125912588388019e-05, + 7.706171459176403e-05, + 7.703835822212567e-05, + 8.402326577652274e-05, + 8.697572925484565e-05, + 8.772432346972892e-05, + 8.923683103603253e-05, + 8.92688174644718e-05, + 8.349913291567647e-05, + 8.536811853900871e-05, + 8.743730256810522e-05, + 7.708513120405768e-05, + 8.644123174690322e-05, + 8.402335868709087e-05, + 9.053668213709898e-05, + 7.887008189300103e-05, + 8.71318281042477e-05, + 8.532634150242304e-05, + 8.958218130601379e-05, + 9.16736916153127e-05, + 8.671341640820174e-05, + 9.029672354472497e-05, + 8.647642505366095e-05, + 9.211567030433796e-05, + 8.278113504900378e-05, + 9.247556408809032e-05 + ] + }, + { + "layer": 19, + "mean_snr": 0.862475838655435, + "per_head_snr": [ + 0.8762409698106083, + 0.8863715506327583, + 0.8519768011546375, + 0.8875097811489521, + 0.8807373725748545, + 0.9230693867391786, + 0.87544024570311, + 0.8685368521381455, + 0.8721096136101076, + 0.835655071180898, + 0.8861680857102083, + 0.8515027927955335, + 0.8310888751525971, + 0.8656680228732543, + 0.8573789378275029, + 0.9017704578947998, + 0.8505184230796196, + 0.8336651314168508, + 0.847106274317621, + 0.8473749860159813, + 0.8219520946349108, + 0.805880303211469, + 0.8504869980502746, + 0.8243679554152308, + 0.8265004959654839, + 0.8708309949151556, + 0.8717870181835241, + 0.8477560930437303, + 0.8314578497114701, + 0.8581703767845182, + 0.98033389443431, + 0.8798131308466267 + ], + "mean_band_energy": [ + 1.6869271987237169, + 1.7722514685444994, + 2.005624710758487, + 2.0486079624049607, + 2.9575126455157696, + 1.7043646705579987, + 1.80317746155401, + 1.908264096487967, + 2.0429717526764932, + 2.9194652909401597, + 3.6281189982033855, + 3.5346042336465326, + 3.644169943432652, + 3.607088163934788, + 3.5771503091912273, + 3.5829897973819143, + 3.550364687674954, + 3.5631735604556085, + 3.618512255920241, + 3.558352192983081, + 3.65217203433566, + 3.547039014177327, + 3.626649324236899, + 3.564111296075681, + 3.5319505746363906, + 3.5598167060344776, + 3.597412721567178, + 3.5787054612744202, + 3.6349603620725097, + 3.541632094350828, + 3.6090972043013116, + 3.60600941096774, + 3.5889043238461604, + 3.611351240020634, + 3.5631143390608164, + 3.5640846344067945, + 3.5205115388901453, + 3.6430129970579475, + 3.486762272517028, + 3.5759998727340836 + ], + "dft_magnitude": [ + 128.3169888235525, + 13.58665135058228, + 9.727466359569977, + 4.518514028545005, + 0.277997584928833, + 3.4790162940622773, + 4.369243995115685, + 3.895813431466767, + 2.2339232193138674, + 1.1217626881890121, + 1.2669247906776682, + 0.9238705826864694, + 0.1334731160795928, + 1.015309280458344, + 1.919261283927911, + 2.636891396936945, + 1.9032881692862524, + 1.8505046381094215, + 1.525488072037793, + 0.9637329178808628, + 0.33514049467761 + ], + "mean_band_dot": [ + 3.512958166993486e-05, + 2.3915526290352316e-05, + 2.8148029673502613e-05, + 3.871156734476245e-05, + 7.517963641703317e-05, + 3.7573176025773595e-05, + 3.2603368600803145e-05, + 2.6096853662238577e-05, + 3.638100374132591e-05, + 7.241819378123185e-05, + 9.626173446122264e-05, + 8.811277307358979e-05, + 9.726799826239583e-05, + 8.913898622608941e-05, + 9.594599085858135e-05, + 9.77711350060417e-05, + 8.622565584346377e-05, + 9.316090436186641e-05, + 9.114789002069298e-05, + 8.535165420653357e-05, + 9.758454739312584e-05, + 9.273848904740588e-05, + 9.924992073138129e-05, + 9.877918937490905e-05, + 9.650748834246772e-05, + 9.697575917471113e-05, + 0.00010481638482815469, + 9.298796358052641e-05, + 0.00010341895099372777, + 9.529509902677089e-05, + 8.840653613333413e-05, + 0.00010361013265622886, + 0.00010094705496612732, + 8.741451972582581e-05, + 9.969875873139245e-05, + 0.00010235581171400552, + 8.708827099326298e-05, + 9.32492662286677e-05, + 8.65447229898564e-05, + 9.74557861809444e-05 + ] + }, + { + "layer": 20, + "mean_snr": 0.8583894958135029, + "per_head_snr": [ + 0.7824946592664682, + 0.8994679200024229, + 0.8998120937846046, + 0.8284520136456721, + 0.8517387878759495, + 0.9007375234888801, + 0.791230860698214, + 0.836270571302667, + 0.8547380459943008, + 0.854113592908399, + 0.8720588284389628, + 0.8969942260578133, + 0.8628529429878612, + 0.905881854791007, + 0.8402997421649767, + 0.8876655367571072, + 0.8582734354253901, + 0.7366100218674201, + 0.8620201846203589, + 0.8250354050768899, + 0.9109896595463166, + 0.8046881825151653, + 0.8835134075216468, + 0.8864812097204128, + 0.8446102010029466, + 0.8345057936374657, + 0.8574013142654198, + 0.9021646057157235, + 0.9643842991271335, + 0.8798004582086032, + 0.8382259242434221, + 0.814950563372471 + ], + "mean_band_energy": [ + 2.0120280788489335, + 2.0911597736636676, + 2.3064065385357857, + 2.4917744214208115, + 3.6273057753176787, + 1.98178690532797, + 2.0123623802064303, + 2.3277010444463757, + 2.4813553656127274, + 3.687073261661188, + 4.3173194365406955, + 4.330091050447931, + 4.494010810268137, + 4.50223778812211, + 4.32284643792282, + 4.2925423681629615, + 4.297035834687759, + 4.439181238968357, + 4.332170888168157, + 4.435372859873923, + 4.360193294474079, + 4.142428535837774, + 4.462349574311279, + 4.490626575619468, + 4.380656325613177, + 4.446544537383486, + 4.437432101974123, + 4.46252964839509, + 4.27966331046598, + 4.375315155460747, + 4.375115637971735, + 4.402116893754647, + 4.27442850018153, + 4.313664310263357, + 4.525444116579643, + 4.404707742079469, + 4.331100718164157, + 4.317706270290576, + 4.322418861261022, + 4.409223031771685 + ], + "dft_magnitude": [ + 156.29542740005743, + 17.11285499456172, + 12.050239784891554, + 6.393998582815708, + 0.34751362427696453, + 4.044595799417216, + 5.697810838108081, + 4.271430636167695, + 3.6470665038253203, + 2.281010790451056, + 1.9566777429378757, + 0.8005179969379762, + 0.4646302204609942, + 1.7278613882509308, + 2.2646000934659796, + 3.303583279604359, + 3.001022066363766, + 2.1480219225521995, + 1.2755433000381244, + 0.6850703525686024, + 0.3921394258457269 + ], + "mean_band_dot": [ + 2.6851746886791266e-05, + 2.836960098875352e-05, + 2.4046440140068624e-05, + 4.430164317810182e-05, + 9.865736839174136e-05, + 2.937679995511644e-05, + 2.5619891761152762e-05, + 2.4025266748139986e-05, + 4.212414492030802e-05, + 0.00010949604271672798, + 0.00010633554252308386, + 0.00011100841757070156, + 0.00011857643073653887, + 0.00011476483473416012, + 0.00010490000597656034, + 0.0001161639188353547, + 9.756422479085816e-05, + 0.00011257347232458414, + 0.00010590623764983319, + 0.00010977361832829047, + 0.00010679818039989186, + 0.0001076636523180241, + 0.00010744388039256596, + 0.00011417874611652221, + 0.00011936408224073604, + 0.00010862414042094314, + 0.0001076765295010773, + 0.0001130304352045641, + 0.00010506792627893446, + 0.00011071368494413037, + 0.00011338735471326797, + 0.00010190682626216585, + 0.00010160794565763356, + 0.00010701703058657586, + 0.0001133049902023231, + 0.00010613514743909036, + 0.00010735213311363625, + 0.00011560672087398418, + 9.005881641428459e-05, + 0.00011217811752430861 + ] + }, + { + "layer": 21, + "mean_snr": 0.8545669419857087, + "per_head_snr": [ + 0.8224629211103305, + 0.7964099932701694, + 0.9015922684937205, + 0.9044434078022967, + 0.9073346334303795, + 0.8978481989612221, + 0.8687889713340932, + 0.8223760537356012, + 0.8280805274467778, + 0.8529136875859187, + 0.8579954177189529, + 0.8570646008396001, + 0.8641665467384899, + 0.8596402763681248, + 0.8430567743855064, + 0.9034397873698947, + 0.8708656135947627, + 0.8267373886189104, + 0.8322568117765698, + 0.7961403569790281, + 0.8509492696110742, + 0.8826999129238129, + 0.8409817455338261, + 0.8613761008876915, + 0.8866236144516486, + 0.7761640774198971, + 0.8405590398331864, + 0.8804790920306357, + 0.8818523527063898, + 0.8155411014368981, + 0.8249357683132461, + 0.8903658308340193 + ], + "mean_band_energy": [ + 2.132036859507328, + 2.2198802585028607, + 2.357311384552666, + 2.5660255141135866, + 3.903872691116794, + 2.0933849630219017, + 2.170988421103802, + 2.293956905811571, + 2.565823947495635, + 3.723739197929662, + 4.670577785954111, + 4.7004350621078395, + 4.953141499521293, + 4.736137424891158, + 4.615712717400186, + 4.6369352483610795, + 4.83173175716233, + 4.629813550156964, + 4.752906592335584, + 4.81307294763482, + 4.787215663782366, + 4.8199455321042235, + 4.804565754575911, + 4.669639363238242, + 4.703187589307403, + 4.75001964582599, + 4.736518675199326, + 4.744873616268679, + 4.777287205113989, + 4.8013211414141335, + 4.851016331048748, + 4.766841731613145, + 4.827832413602178, + 4.453842050860463, + 4.650735087189553, + 4.738958819366881, + 4.780250415452763, + 4.502984954294492, + 4.587746015197057, + 4.819463644583745 + ], + "dft_magnitude": [ + 167.94173037872048, + 19.672532922923402, + 13.321581190121249, + 6.280242642930238, + 0.821692400293735, + 5.256720212022492, + 6.377267533284077, + 4.962599618225353, + 2.8857836910634878, + 2.5281202488873546, + 2.0201577427932262, + 1.430947658425258, + 0.9076336758750668, + 1.6803089874778603, + 3.2647255707577933, + 3.1233115717082103, + 3.266124327334038, + 2.62530694225697, + 1.6322742476177599, + 0.9480904728539029, + 0.9791872345175676 + ], + "mean_band_dot": [ + 3.6248242292913356e-05, + 3.920868764453189e-05, + 3.145759954179539e-05, + 4.801297322956088e-05, + 0.00010461184417636106, + 4.711734697480097e-05, + 2.901018250440757e-05, + 2.4905628282567705e-05, + 5.2908165616827316e-05, + 9.676930121713668e-05, + 0.00011597142620303199, + 0.00011378165922337756, + 0.0001241395223132713, + 0.00011232607339479725, + 0.00011784624450683623, + 0.00011344303684381885, + 0.00011097224817149256, + 0.00011933505604986292, + 0.00011984109253262431, + 0.00012019132000204991, + 0.00011284605537866809, + 0.0001185055200608076, + 0.00012127076233809933, + 0.00010762014368310701, + 0.00011553732274478537, + 0.00010806857267198212, + 0.00011735718235286185, + 0.00010895582681769155, + 0.00011926792876693067, + 0.00012068042756254726, + 0.00011471997918306441, + 0.00011445537888903344, + 0.00012037709248033935, + 0.00010752125620001605, + 0.00011777541521951208, + 0.00011974057315455866, + 0.00011421551021157936, + 0.00010768110529397752, + 0.000119081705042845, + 0.00010232793929390028 + ] + }, + { + "layer": 22, + "mean_snr": 0.8421583850324771, + "per_head_snr": [ + 0.8282499757971489, + 0.8957871695056052, + 0.7984379077300858, + 0.7757652583241134, + 0.8238934475939375, + 0.8961085183185136, + 0.8153843135330773, + 0.836487546386196, + 0.8193254670561627, + 0.9251633914026199, + 0.764952136671976, + 0.8389723493905333, + 0.8738402815333768, + 0.8692897806656877, + 0.8381551498713657, + 0.8791751917055637, + 0.8659855810408802, + 0.8827481065387455, + 0.7819936755732895, + 0.8636738950083889, + 0.8280230428504182, + 0.85520543262865, + 0.8066424270386994, + 0.8128857693709168, + 0.8284002494299719, + 0.866528043443762, + 0.8115225770326285, + 0.8599809269058494, + 0.8184197553030659, + 0.8569043430304323, + 0.9107274746302879, + 0.8204391357273182 + ], + "mean_band_energy": [ + 2.4760807383095025, + 2.620310638981807, + 2.8227553450743796, + 3.108312067864481, + 4.661670819481577, + 2.4791980104888665, + 2.5664098467246745, + 2.9132165582419343, + 3.3715170104621524, + 4.611154025921122, + 6.051347137773714, + 5.894116271531492, + 5.8863199741779155, + 6.18219258628241, + 5.938521217622027, + 5.93877182181473, + 5.725413811854905, + 6.011591838408657, + 5.87995948414488, + 6.068611048351923, + 5.95673083140009, + 5.954349948287357, + 6.246732692598231, + 5.784845844000668, + 5.676603220195415, + 5.817134458604712, + 5.999564592976513, + 5.54009950145879, + 5.927437247542276, + 6.055349857726216, + 5.999371367582429, + 5.917556900872674, + 6.047768967429139, + 5.985139465329856, + 5.811796774734368, + 5.584350867237444, + 5.987116405713218, + 6.178466503131145, + 5.858992158651927, + 5.887063520098849 + ], + "dft_magnitude": [ + 209.42394137908445, + 24.731662636827625, + 18.443249353175275, + 8.502938820652457, + 1.2923408534928569, + 6.400625505437113, + 8.160486179308315, + 7.155321623538974, + 4.249128368353554, + 2.187608413533057, + 1.6366800541117157, + 2.0676368663461897, + 0.6587393799814797, + 2.839098932974339, + 3.313518220138091, + 4.258518719058163, + 3.4845056320042604, + 2.753893893122654, + 3.3898983729266505, + 0.6242245277844407, + 0.3602779098142008 + ], + "mean_band_dot": [ + 4.609241899515837e-05, + 4.024573519245678e-05, + 3.239164500428159e-05, + 6.960581686143993e-05, + 0.00012822724095258307, + 3.8500854475387304e-05, + 3.38217064751234e-05, + 3.330732125732539e-05, + 6.020510393227596e-05, + 0.00012001372876966345, + 0.0001381596810915653, + 0.00014316134878527005, + 0.00014484773560070608, + 0.00014703137825335943, + 0.00015536427347342399, + 0.00014137835814835852, + 0.00014393276705959576, + 0.00014481398492875998, + 0.00014905334965078512, + 0.0001313388134576599, + 0.00014559017411102106, + 0.00014654817869086398, + 0.0001379823144816328, + 0.0001444490726555614, + 0.00012459742292776356, + 0.00014390401104265037, + 0.00013480084626280587, + 0.00012797126819350524, + 0.00013947187085250332, + 0.00013849547098629956, + 0.0001383517239901266, + 0.00014418650789593815, + 0.00015485470803469072, + 0.0001346490547120993, + 0.0001364177531399946, + 0.0001416881790646585, + 0.0001433370992685923, + 0.00014677992530778283, + 0.00013894771348077484, + 0.00015861282581681733 + ] + }, + { + "layer": 23, + "mean_snr": 0.833252794795752, + "per_head_snr": [ + 0.8054886665561818, + 0.7958916696809266, + 0.8438527872963812, + 0.760904268603814, + 0.8079407145063227, + 0.8476437377365855, + 0.888208748235441, + 0.8495979478773981, + 0.8159971897306247, + 0.8737525437327706, + 0.8807374808992078, + 0.8752623183109267, + 0.8486017067600641, + 0.8170765982881265, + 0.7934722736807671, + 0.808484876888174, + 0.7929575307347092, + 0.8410487889317301, + 0.8416113983294163, + 0.8639800190608444, + 0.8399852386550956, + 0.8391719855438771, + 0.7707547202901681, + 0.8575870401914141, + 0.8419415586299934, + 0.8186501659827292, + 0.9515932782984018, + 0.7730324920585009, + 0.840283437120723, + 0.8450747360756339, + 0.781349819644035, + 0.8521536951330813 + ], + "mean_band_energy": [ + 2.9161165020327897, + 3.0750290924067634, + 3.374628393362824, + 3.832162721278433, + 5.488432495714857, + 3.0480034121165955, + 3.0954131595776877, + 3.3113172614399087, + 3.7809750693436914, + 5.2562877262990515, + 7.533754265505468, + 7.289481697139301, + 7.504890273774118, + 7.265007271030494, + 7.645694877974916, + 7.196924189113074, + 7.311176540269246, + 7.6614859222912886, + 7.221340949053775, + 7.347347026100334, + 7.4070843373477295, + 7.7932148100045096, + 7.287033474994851, + 7.469288997657804, + 7.864917339793322, + 7.483693594589617, + 7.1115310379358405, + 7.4562170575003615, + 7.4307929472449095, + 7.627093709487261, + 7.419257667732478, + 7.1646827680172125, + 7.34653957910699, + 7.701948794751098, + 7.344489205344291, + 7.534697924120131, + 7.432072120430935, + 7.322607948573201, + 7.4310971778840464, + 7.367284948034295 + ], + "dft_magnitude": [ + 260.15101428637547, + 33.892345993122944, + 23.310505300452327, + 11.72696231115255, + 0.9494043309851504, + 8.098876651331048, + 10.387725326065164, + 8.37875365301185, + 5.908680216940286, + 3.6749874438173253, + 1.993282455212846, + 4.03596643278236, + 1.1630564714687257, + 3.212749148947653, + 5.286010443899021, + 4.060167289137063, + 4.254800348021021, + 2.0238168085172643, + 3.5439095142631483, + 2.9879920230854053, + 0.2565394575259745 + ], + "mean_band_dot": [ + 5.853488414402364e-05, + 3.517939467201358e-05, + 2.6912497696685025e-05, + 6.018049748490739e-05, + 0.00014320027211169872, + 7.615977261252734e-05, + 4.684483077994627e-05, + 2.6872312670889186e-05, + 5.3049845860186906e-05, + 0.00012442451449032888, + 0.000154379967375462, + 0.00015061744920785716, + 0.00016311615031554537, + 0.00013808091285909545, + 0.0001705171401454209, + 0.00016094031770990115, + 0.00016135477933403304, + 0.00014480543109129937, + 0.0001551127728362189, + 0.0001471340139687527, + 0.00014944743761589053, + 0.00016242218783872885, + 0.00015383709055640792, + 0.0001868272000592697, + 0.0001600405500994384, + 0.00016185162282909, + 0.00015916280524379544, + 0.0001590600682675358, + 0.000141231137649811, + 0.00015588399460284564, + 0.0001443797296701632, + 0.00015794158944117955, + 0.000139585640590667, + 0.0001601769336957659, + 0.0001507478469648049, + 0.00016236729920819927, + 0.00015027819790702784, + 0.00015975549636095814, + 0.0001699096823358559, + 0.00015482448848160856 + ] + }, + { + "layer": 24, + "mean_snr": 0.8374343236263044, + "per_head_snr": [ + 0.8851791344332272, + 0.7921744423381141, + 0.8624022245768919, + 0.8332806680410514, + 0.8213714240360253, + 0.804410717202818, + 0.7868206333078337, + 0.824366192376569, + 0.8186647255518217, + 0.8212655785107951, + 0.8072184070146207, + 0.8766797013498396, + 0.8452068508227303, + 0.8425403294480768, + 0.8771572319676172, + 0.893034521280235, + 0.8493823010015225, + 0.8978805013483605, + 0.8153851478351979, + 0.7898416874298427, + 0.7905041735485923, + 0.8853788460619956, + 0.8105909721954011, + 0.8327178962402394, + 0.8497888531389614, + 0.8161559109129225, + 0.8139122313475493, + 0.8042828858241617, + 0.8628068163602747, + 0.8843065657808453, + 0.8395069082888217, + 0.8636838764687836 + ], + "mean_band_energy": [ + 3.906407899585986, + 4.009127072843851, + 4.022641332222336, + 4.374528614719875, + 6.84489166085827, + 3.7747853450368014, + 4.042814065476271, + 4.30444797202809, + 4.442031199170935, + 6.75743435470921, + 9.336961969442408, + 8.986617469811986, + 9.252420754585668, + 9.6220101667426, + 9.13649067112724, + 9.531707566266574, + 9.503752038710656, + 9.188952148463844, + 9.401409833162598, + 9.521092181166905, + 9.45692867668742, + 9.42179607002966, + 9.876991639051807, + 9.671810800165584, + 9.251144331102891, + 9.310004108806002, + 9.718612600739801, + 8.887391703902434, + 8.783344964632972, + 9.192526581307744, + 9.207229245423873, + 9.184058404440986, + 9.602659703719393, + 9.530561017516469, + 9.129598773824915, + 9.347216147202618, + 9.150331344559715, + 9.32434223442107, + 9.306776600575079, + 9.148098802577978 + ], + "dft_magnitude": [ + 326.46194806682047, + 43.20558024966891, + 29.936955971233928, + 14.723050966735904, + 1.673445016030191, + 8.378925672083504, + 12.855134267564404, + 11.076186684436198, + 5.118155227408806, + 2.938475071344511, + 4.1105899527972465, + 1.837920494048629, + 2.3968670317684535, + 5.239544307285299, + 6.574571992092836, + 5.977353258548662, + 4.32850760437787, + 4.215307449101204, + 4.248256339496282, + 2.100950345127612, + 0.28493054249995 + ], + "mean_band_dot": [ + 8.857366099164212e-05, + 6.520657927922002e-05, + 4.4745391414835466e-05, + 7.000932163805374e-05, + 0.0001793169594861865, + 8.941211717683475e-05, + 6.0953364953775226e-05, + 4.496992755775864e-05, + 5.4466362348648546e-05, + 0.0001817247414464873, + 0.00016567845882491385, + 0.000171665545212818, + 0.00018469332767381277, + 0.00018257876897678215, + 0.00017189482657613555, + 0.00016953059798652248, + 0.000180316292153293, + 0.00016199969678609705, + 0.00016243100171777768, + 0.00016315840684910651, + 0.0001794191887711349, + 0.0001847065355150335, + 0.00018686947105379654, + 0.00016771499110745938, + 0.00020432227829587644, + 0.00019607594849730954, + 0.00016981620101432778, + 0.00017408191183676536, + 0.00016962717168098607, + 0.00016184637888727593, + 0.00017591590622032527, + 0.0001673634262942869, + 0.00016230036490014755, + 0.00017244802284039907, + 0.00018466138637904803, + 0.00017916489632625597, + 0.00017085824779314857, + 0.0001697557486295409, + 0.00018464182730895117, + 0.0001680841665802291 + ] + }, + { + "layer": 25, + "mean_snr": 0.8350603127059958, + "per_head_snr": [ + 0.7992762166924854, + 0.8011383217446861, + 0.8446968186593706, + 0.907212658403447, + 0.7802439048292968, + 0.8274972144095125, + 0.9374786484236629, + 0.8198912109916427, + 0.7959651607497994, + 0.8162202736755777, + 0.8228050114630943, + 0.7826079392372168, + 0.8110571687564763, + 0.7975372152551854, + 0.7895825428241567, + 0.912874222919985, + 0.8551426468769048, + 0.8665306997612272, + 0.7951742560959315, + 0.8702764025675754, + 0.8392372616238784, + 0.8642539768978993, + 0.8621201739963306, + 0.895388557177731, + 0.8449701499185733, + 0.8385861072376675, + 0.877892590240633, + 0.8213215350897594, + 0.876069325277033, + 0.8034480187351538, + 0.7818874208728889, + 0.7835463551870815 + ], + "mean_band_energy": [ + 3.9794023917731574, + 4.2658879817070625, + 4.334862018794489, + 4.910552039882421, + 7.025076857020974, + 3.9772527977197605, + 4.2778790405045175, + 4.398335339568999, + 4.909688747083428, + 6.934025537196541, + 9.560433215856014, + 9.68298893623475, + 9.963920987164856, + 9.763609267888983, + 9.699258496879857, + 9.864671631391191, + 9.451677935696454, + 9.952678092765117, + 9.664982337093726, + 9.515751929677297, + 9.783408509552967, + 9.852886377441923, + 9.77063654297113, + 9.820080621965587, + 9.923100709893601, + 9.699533219467556, + 9.79395077180394, + 9.750389032200667, + 9.41478422591812, + 9.827677568665262, + 9.359250798929109, + 9.472884232306473, + 10.16681370882247, + 9.787954222376172, + 9.933700706168974, + 9.661187845217412, + 9.30998550093779, + 9.684114825652742, + 9.78704438050645, + 9.484370878946091 + ], + "dft_magnitude": [ + 340.416690261644, + 43.958601439228964, + 30.9377438801282, + 15.161649328029265, + 1.853342242670906, + 9.205165172455459, + 12.932897415796152, + 11.428949997271275, + 6.119027988991943, + 4.126217896900728, + 3.6067579776099574, + 3.187494481007177, + 0.7307650373331483, + 4.21358994482034, + 5.208295378510355, + 5.297606662653501, + 5.455565508448903, + 4.755302147132573, + 4.14623493009566, + 2.111554497455442, + 0.1969744948999903 + ], + "mean_band_dot": [ + 0.00010331056751056167, + 6.76256062035918e-05, + 3.508062050059379e-05, + 6.51337999926227e-05, + 0.00019044756008952394, + 0.0001009357642260511, + 7.10939845816938e-05, + 3.380188736059608e-05, + 6.726776728100958e-05, + 0.0002007091525456417, + 0.0001714043746233074, + 0.0001923817007764228, + 0.00018494222108529358, + 0.00021248908474262866, + 0.00020932136631017788, + 0.0001908762439597922, + 0.0002179070636657343, + 0.00017324882137472738, + 0.00019057854151469659, + 0.00019596820318383838, + 0.00020933575356139048, + 0.00019291696842174133, + 0.0002020358825120638, + 0.00019289419033157172, + 0.0001995995484321611, + 0.00018876536785228384, + 0.00020064678615199226, + 0.00018876374193723674, + 0.00018669383275664585, + 0.00021103015901644543, + 0.00017078107957786413, + 0.00020392224396346137, + 0.000189821745016161, + 0.00019087534369646164, + 0.00017993355006638015, + 0.00019591776208471853, + 0.00019104054827039362, + 0.00018118053319540194, + 0.00019241423565290457, + 0.0002088489835841756 + ] + }, + { + "layer": 26, + "mean_snr": 0.8309293399839746, + "per_head_snr": [ + 0.8552053201907358, + 0.8700522119347923, + 0.8712336980235839, + 0.8772256541816135, + 0.8419376690733559, + 0.7739725759119618, + 0.7876810717540736, + 0.781968191850262, + 0.7607030553032614, + 0.8145389438520951, + 0.85916827034589, + 0.8543452774094863, + 0.8272122764148044, + 0.871369661627124, + 0.843282246592864, + 0.7831813814473697, + 0.7725497596425093, + 0.8349711793983845, + 0.8120772177637119, + 0.8578478388729406, + 0.8268737349555885, + 0.8321716214403296, + 0.8081439908486158, + 0.7973356465904125, + 0.8624438441700021, + 0.8697959371651943, + 0.921478395885605, + 0.7942240194785551, + 0.8246299158497561, + 0.8067751208725226, + 0.8427036344949563, + 0.852639516144827 + ], + "mean_band_energy": [ + 4.172625247722237, + 4.457745660222738, + 4.605030463260245, + 5.2380857000800445, + 7.670036228293909, + 3.9821028609015734, + 4.107750314739873, + 4.699665565604336, + 5.353109555854569, + 7.78885105756876, + 10.609601043660374, + 10.214615283761727, + 10.640415566364668, + 10.677546695695858, + 10.231658416137597, + 10.768895468529943, + 10.585688431635877, + 10.602318615484418, + 10.55455683258465, + 10.48684870918173, + 9.927019786592576, + 10.542962632600258, + 10.46739208177864, + 10.445990502533517, + 10.220506265980923, + 10.38833770040995, + 10.030683881211623, + 10.027054166140104, + 10.489201905922446, + 10.236750539880665, + 10.673274151035315, + 10.237183354043609, + 10.708648219061768, + 10.552718716243923, + 10.399016333303628, + 10.341354274388088, + 9.993155053792007, + 10.598751069412977, + 10.350888823106247, + 10.422388514947338 + ], + "dft_magnitude": [ + 364.50042568967075, + 46.40692997633959, + 35.5727077464204, + 15.592477252279084, + 1.2396665564979672, + 9.189529954624746, + 15.218720766962655, + 12.51142037569088, + 7.87474373881099, + 4.522964473696618, + 3.8049017096314595, + 2.9337426115156484, + 0.722519319444746, + 3.5509723441446295, + 7.596030948605816, + 6.6343828696915645, + 4.59338936132533, + 5.4346743389233945, + 6.288458960702177, + 1.8704450396364731, + 0.9199084855924298 + ], + "mean_band_dot": [ + 0.00012695347047610994, + 8.697112796198783e-05, + 5.2004907820446535e-05, + 8.9200183003868e-05, + 0.00024389564259763577, + 0.00010248162564039375, + 7.055777909954483e-05, + 5.427358347844801e-05, + 7.552965234935984e-05, + 0.0002351713501980157, + 0.00022108288330855433, + 0.00021393837759546837, + 0.00022226974961085945, + 0.00023462297772312014, + 0.00023561033158330246, + 0.000236063688566901, + 0.0002335211354875355, + 0.0002229040497695678, + 0.0002032316320764948, + 0.0002374927950768324, + 0.00023671062717767197, + 0.00025051953093679913, + 0.0002070210699912423, + 0.00023763500157656376, + 0.00021827033806403055, + 0.0002292875751663814, + 0.0002118463207807509, + 0.00021251143461995526, + 0.0002487745012331289, + 0.0002038565770362766, + 0.00022810502641732458, + 0.0002163434634894656, + 0.0002517364566301694, + 0.00023188106847555907, + 0.00024101637673084038, + 0.00021054490971437193, + 0.00022075593608406052, + 0.0002419903542431712, + 0.00022566776542589653, + 0.0002048235750407912 + ] + }, + { + "layer": 27, + "mean_snr": 0.8154280343067282, + "per_head_snr": [ + 0.8654605214404797, + 0.8277688816364498, + 0.8250399100338208, + 0.8745544646993296, + 0.7830743895338165, + 0.7663240494108025, + 0.765937205809676, + 0.8005576586429668, + 0.7667919357955675, + 0.8379939931170588, + 0.8721745149882422, + 0.7859359985628205, + 0.8584309303900906, + 0.8113728118207294, + 0.7661074791000612, + 0.7905179256694049, + 0.8096833757531088, + 0.8374402741318602, + 0.7369689719862232, + 0.7335690787197313, + 0.7981752868964701, + 0.8842258448856113, + 0.7615445145660769, + 0.7794070251855952, + 0.8378406456780075, + 0.9136777962599107, + 0.8466852500867104, + 0.9520250212474201, + 0.8309280951995747, + 0.7830710406002157, + 0.7392911038629584, + 0.8511211021045119 + ], + "mean_band_energy": [ + 3.6796504574221682, + 3.779642090489872, + 4.08416379571535, + 4.4384810319260835, + 6.3791980211030355, + 3.7873737412455437, + 3.8839226113131775, + 4.2546640980350325, + 4.600631278699752, + 6.374611494768354, + 10.283330893733755, + 9.839048062982418, + 10.30185516134629, + 9.970280547078325, + 9.70685919840967, + 9.597096459259902, + 9.967313883462198, + 9.992218212013045, + 10.113931325204115, + 9.830178361706219, + 9.61354558883127, + 10.104277047013465, + 9.805142143508183, + 10.293057272459468, + 9.708924079392892, + 10.102448382191241, + 9.672810768542423, + 9.837706558735624, + 9.805059699490382, + 9.790723935080452, + 9.951531084167334, + 10.272304321952529, + 9.887830956476424, + 9.857167746866194, + 9.69504181744712, + 9.902375584159408, + 10.05730367279228, + 10.088041839792874, + 9.738374202190428, + 10.363772898555037 + ], + "dft_magnitude": [ + 343.4118903255593, + 47.95019129031678, + 34.93548065852127, + 17.596355977955948, + 1.0572838159489522, + 10.516183774242837, + 16.197130040225108, + 10.307279670951669, + 4.484770518356918, + 3.99061720124374, + 5.610144138512724, + 4.890669045868198, + 0.5583324114971315, + 3.43766462999093, + 7.122329796181454, + 7.545644279199428, + 4.03914299562701, + 4.406621370909894, + 5.318137601506016, + 4.765503223592695, + 1.5390490470628322 + ], + "mean_band_dot": [ + 1.8231437093163545e-05, + 4.2128645714001325e-05, + 3.22513031278504e-05, + 5.690371215507639e-05, + 0.00015311894890146505, + 2.6190995913566443e-05, + 4.597659104206286e-05, + 2.0266127216927993e-05, + 4.3804623783216814e-05, + 0.00014321193890509676, + 0.00015097203004188484, + 0.00015321322687213978, + 0.00015671135126922307, + 0.00016988685929391066, + 0.00014424530221504028, + 0.00015398938171529153, + 0.00017925737561199636, + 0.00016189477106536292, + 0.00015584952998324295, + 0.00016427240416305724, + 0.00014838114682333978, + 0.000138042211665379, + 0.00017007353685585255, + 0.00013063172741567544, + 0.00016016718400351236, + 0.00013735103070757761, + 0.00016387204323109472, + 0.00015788802472798126, + 0.0001378696946176206, + 0.0001503271451838373, + 0.00014970890024414984, + 0.00017355336576656554, + 0.0001428380996458145, + 0.00016468085527776563, + 0.00014643683807662454, + 0.00015691029561821783, + 0.00019130273831251545, + 0.00014774110504731656, + 0.00015774635498928542, + 0.0001625446643856776 + ] + }, + { + "layer": 28, + "mean_snr": 0.8133661844037099, + "per_head_snr": [ + 0.8438268454710783, + 0.8819633238650806, + 0.7979629522586341, + 0.8002849937678842, + 0.7539351471129979, + 0.7948209300281263, + 0.8591799926302975, + 0.7925595214680022, + 0.8264034291422212, + 0.8735510648466024, + 0.7910812911605514, + 0.6934293146203583, + 0.9025566090025324, + 0.7583568687002298, + 0.888137530350871, + 0.7325525700246835, + 0.8259058031522621, + 0.8280460784461465, + 0.7410149253775694, + 0.8572170042572803, + 0.8265360523445615, + 0.7817941556419555, + 0.7674814841091542, + 0.8478445323551874, + 0.8056094980154287, + 0.8555552142020382, + 0.7649480820520775, + 0.874777965061107, + 0.8038244560013726, + 0.7975737843352851, + 0.7401766213921082, + 0.9188098597250296 + ], + "mean_band_energy": [ + 3.9616362585466414, + 4.028470214263168, + 4.212363994123134, + 4.8379341721628535, + 6.541721703439181, + 4.077535134840371, + 4.0336435973632465, + 4.150020904209556, + 4.942477461347706, + 6.396471308882653, + 10.845369668898979, + 10.34492140789165, + 10.132567712924399, + 10.596201120598124, + 10.123585476993087, + 9.968381251958197, + 10.42137253384192, + 10.29026475017291, + 10.3886154306982, + 10.033106875898204, + 10.557013045041993, + 10.430436598395763, + 10.140155756952414, + 10.772038772629996, + 10.214964510231143, + 10.647175310343155, + 10.844277461400878, + 10.93027164072172, + 10.46612417653375, + 10.669162701674693, + 10.663740573324615, + 10.476333595153163, + 10.1717569915881, + 10.301612394903099, + 10.387456715816324, + 10.488983985623449, + 10.300853471216636, + 10.972149914750236, + 10.402900303144385, + 10.340662818313966 + ], + "dft_magnitude": [ + 360.5047317168137, + 52.213808992553226, + 35.01742717589917, + 19.023864152107258, + 1.603485253924849, + 12.63990915583751, + 15.505302288172151, + 11.792601418771865, + 4.644095971760678, + 4.304122102601633, + 5.85088153455619, + 3.709488870640434, + 2.5276255750554784, + 5.2317398104587935, + 5.880802418704778, + 5.862494942637199, + 2.2531399009856035, + 5.6505991267621445, + 4.946507466216644, + 4.866500701668277, + 0.9995380299602346 + ], + "mean_band_dot": [ + 4.8704818095757214e-05, + 5.0815272743420785e-05, + 4.234038464119294e-05, + 6.640993785822502e-05, + 0.00018963150068884718, + 7.825536836207901e-05, + 6.78453003274626e-05, + 3.0018907585827042e-05, + 6.915482284739482e-05, + 0.0001815773587622971, + 0.00021043445863142552, + 0.00021470261071954153, + 0.0002265463122853362, + 0.00019972313725702406, + 0.00020313189936587147, + 0.00021823166919148206, + 0.0002076920104855162, + 0.00023090833674359603, + 0.00019870896024940524, + 0.00020499366391959478, + 0.00019571394916511055, + 0.00021548211589106356, + 0.00023633008613614952, + 0.00018943211325677113, + 0.00021173415897237646, + 0.0002131555273081176, + 0.00020903811052903616, + 0.0002040381170445471, + 0.00022547220444266717, + 0.00019877191157320338, + 0.00021002575249440272, + 0.0002165590852882815, + 0.00020886219208477997, + 0.00019499417392125908, + 0.0002025152431542665, + 0.00020044560822043425, + 0.00020124750324157506, + 0.00021248449913855436, + 0.00021869151951250387, + 0.0002005565587523961 + ] + }, + { + "layer": 29, + "mean_snr": 0.8233788290642468, + "per_head_snr": [ + 0.8858479892887824, + 0.773858477103753, + 0.8495516502762177, + 0.8087992167417283, + 0.804797121487775, + 0.824380171057174, + 0.7495710186358058, + 0.8670272965181477, + 0.8663975477854774, + 0.8284412888291611, + 0.795556428826593, + 0.8221916270788054, + 0.7611696735349502, + 0.8899922369245007, + 0.8748278372749716, + 0.8838598833626042, + 0.8399336727177193, + 0.7920399651884675, + 0.7828339217134835, + 0.8129348263939848, + 0.8301756022725372, + 0.8558140200498929, + 0.8090175948556841, + 0.8410401724778327, + 0.8314971277798858, + 0.7566194899094691, + 0.9120564002087618, + 0.7806603744906203, + 0.7946947018835739, + 0.7493197260299868, + 0.8006052370695469, + 0.8726102322879973 + ], + "mean_band_energy": [ + 4.1065967967404315, + 3.872061018896615, + 3.8060368050139, + 4.213326248609886, + 5.7564965948352915, + 4.044536032897547, + 3.986196571012883, + 3.8565826124719154, + 4.594900582662044, + 5.734812536043136, + 9.461300030538466, + 9.262857586543793, + 9.107718262970153, + 9.14896068984047, + 9.297100621862377, + 9.318221770211895, + 8.586352321433244, + 9.5659260137184, + 9.355174119836052, + 9.265062504885465, + 9.412857462961561, + 9.496576353571644, + 9.496539195304692, + 9.362862341849949, + 9.33572091540588, + 9.007255609460884, + 9.171365736547601, + 9.550078680439984, + 8.991633397084698, + 9.475615307059385, + 9.496841059961701, + 9.176377508726583, + 9.4501949621492, + 9.078145551651035, + 8.990617516870127, + 9.50628336711246, + 8.893573956617939, + 9.29007449533856, + 9.075996137878915, + 9.245364524222484 + ], + "dft_magnitude": [ + 321.84419380123927, + 44.78332170629453, + 30.351803633503668, + 14.741898607413063, + 1.7687112918096444, + 10.088334754893328, + 12.288188319959787, + 9.457075599919749, + 4.339491314962309, + 1.870539530840283, + 5.6182772259141025, + 3.9939118584587137, + 1.1863218924373693, + 4.679775470176821, + 4.55007114397253, + 3.1073104941591723, + 3.1589425266389397, + 4.231834616409174, + 5.437830640495969, + 3.109632446872109, + 1.0977677058649533 + ], + "mean_band_dot": [ + 8.262507163863119e-05, + 4.5982666074451116e-05, + 4.7133498094353866e-05, + 6.735587201376347e-05, + 0.00016311714998664686, + 8.472180867897806e-05, + 8.450194039824056e-05, + 3.4987047922641064e-05, + 4.548728913960076e-05, + 0.00016415725947354075, + 0.00018199805672338695, + 0.00019885483216057763, + 0.0001903726468299283, + 0.00019728593650825132, + 0.00017670132574494344, + 0.00017328494650428186, + 0.00018560998155408014, + 0.00016513804889655147, + 0.00018477804028407254, + 0.00018637611741496586, + 0.0001748016088640725, + 0.00018032006187240776, + 0.00017685678417365123, + 0.00020457800719668738, + 0.00018282011142218833, + 0.0001677439345257881, + 0.00017745728209774824, + 0.00019386892487318618, + 0.00019419809570990764, + 0.00019103200602330615, + 0.00018791688397072902, + 0.0001761288378475001, + 0.00019057913596043363, + 0.0001790321165117348, + 0.00018898042890214128, + 0.00019871985550707903, + 0.00015700991200446874, + 0.00017623435055611473, + 0.00016931577262084828, + 0.00018265336984768504 + ] + }, + { + "layer": 30, + "mean_snr": 0.8254775358471589, + "per_head_snr": [ + 0.8109443963206168, + 0.8328769738344488, + 0.7937006952119384, + 0.8553319945641142, + 0.8388628173452016, + 0.8004900772510702, + 0.8274919676470605, + 0.816325147621604, + 0.841018649004737, + 0.8853176434190936, + 0.865268114521248, + 0.8900695972590096, + 0.9244513803850922, + 0.8167688529268136, + 0.8338502259033931, + 0.7947616558284284, + 0.7823439399099038, + 0.8731813207873794, + 0.7403928283683906, + 0.7137689111579328, + 0.8332518132328316, + 0.8394358445289232, + 0.8575217185707773, + 0.8306705311074838, + 0.7625052850780172, + 0.7761003504013174, + 0.9345543580553086, + 0.812930788672294, + 0.8084592351821152, + 0.820816924482719, + 0.7549054550434194, + 0.8469116534864045 + ], + "mean_band_energy": [ + 3.901334074153278, + 3.9288180219747786, + 3.9758735027357206, + 4.081359836135495, + 5.40478320733338, + 3.7223554852671974, + 3.765498506834439, + 3.8567043223523507, + 3.956264631304146, + 5.621669173032791, + 9.496476859041936, + 8.999033009130335, + 9.438151899337806, + 9.54067370055272, + 9.256244965390643, + 9.274331711839157, + 9.863213192338826, + 9.275180367162472, + 9.802189718780067, + 9.633074051377081, + 9.336230187508741, + 9.407652608218635, + 9.61436967678873, + 9.354383987280185, + 9.595063869854604, + 9.470151992450674, + 9.484407596763155, + 9.692772876105494, + 9.681318945956384, + 9.049635237148268, + 8.99933760939737, + 8.98698508290611, + 9.144439833791475, + 9.411756093869137, + 9.24746474677868, + 9.578669610677343, + 9.972345487939144, + 9.34089827687462, + 9.530814780681965, + 9.093963198930103 + ], + "dft_magnitude": [ + 324.78589193599544, + 47.469696156097754, + 33.10688140417744, + 17.223793873355806, + 2.2379711454402225, + 10.759926603766651, + 11.883752887215849, + 9.098756687271788, + 3.026066722381973, + 2.1623679443017263, + 4.753831512833029, + 4.396422980556729, + 1.2565638090694253, + 3.05593603212798, + 8.0586231156937, + 4.56748483058271, + 2.4255354040120887, + 2.482613069158912, + 3.0864684181419157, + 3.1123597841618533, + 2.1457546494255553 + ], + "mean_band_dot": [ + 3.526891156298006e-05, + 2.6396286102681188e-05, + 3.0182981674897746e-05, + 5.124884874021516e-05, + 0.00015801408371771686, + 1.6171188065072784e-05, + 3.536919634825608e-05, + 4.121566665844512e-05, + 4.2842141135679425e-05, + 0.0001371226097035105, + 0.00015937407806632114, + 0.00019130563086946497, + 0.00019027122648367366, + 0.00016895848807507718, + 0.00015876042189120202, + 0.0001819277123559005, + 0.00016858854523889028, + 0.0001450525873224251, + 0.0001873186308330332, + 0.00016616919992884507, + 0.00017858760227227322, + 0.00014801968854953886, + 0.00016551952677446027, + 0.0001946393499565602, + 0.00017585917571523172, + 0.00018179127446273923, + 0.00017205728167937198, + 0.00015754642172396415, + 0.0001666313121859275, + 0.00018213360255003863, + 0.00016828003465434445, + 0.0001751473286731198, + 0.0001711350636696807, + 0.00014237351518886499, + 0.00017138968232757179, + 0.00016396819472674906, + 0.0001722920193060418, + 0.00017082667930026218, + 0.00018081502594213816, + 0.00015053703642706757 + ] + }, + { + "layer": 31, + "mean_snr": 0.8436582983404558, + "per_head_snr": [ + 0.826807112973679, + 0.8043417658414708, + 0.927025198318659, + 0.8375512767212084, + 0.8262673878741239, + 0.7822677276825802, + 0.8553624324139243, + 0.8665690106475068, + 0.8949534030352144, + 0.7778569892450706, + 0.8284229460136346, + 0.8255335456262556, + 0.8208459539211098, + 0.8446734427084445, + 0.9357638724305875, + 0.9197717053072572, + 0.6478448530122692, + 0.8756592131114502, + 0.8022679628837672, + 0.7924766381337068, + 0.9022088993310673, + 0.796511748750775, + 0.8926920701825509, + 0.8242779065925055, + 0.8296900151729227, + 0.850990095922438, + 0.86072490887921, + 0.9260653987094833, + 0.8205535885975581, + 0.9501851821965743, + 0.7784865351338357, + 0.8724167595237435 + ], + "mean_band_energy": [ + 2.8344893148529633, + 2.5839884105435966, + 2.6478162877199867, + 2.523220369634076, + 3.846452537771333, + 2.779654323849207, + 2.70109665708214, + 2.490061178134345, + 2.893023939496795, + 3.8829262743302877, + 6.377225334646574, + 6.3449073122829756, + 6.458055638396142, + 6.243274132217949, + 6.381057627407538, + 6.451168072892712, + 6.085612722298987, + 6.326222436976643, + 6.44037319793876, + 6.298006539167622, + 6.346156751280826, + 6.408966791937794, + 6.452412969174461, + 6.217768477507561, + 6.705225050736875, + 6.529998508775164, + 6.229691684746642, + 6.444056540422558, + 6.41906145501288, + 6.318642996090026, + 6.224823925071872, + 6.08875841070421, + 6.031061846203143, + 6.556988063144782, + 6.148393192547723, + 5.771611530406161, + 6.400550272316511, + 6.0749034447491335, + 6.224040080301522, + 6.426390787816345 + ], + "dft_magnitude": [ + 218.60813508658683, + 31.406031377852727, + 20.501179648389407, + 11.547286290109108, + 0.35370680266115584, + 6.328532220053017, + 8.743232319315965, + 6.917381194849197, + 3.4603841937770845, + 2.102510541487738, + 2.256027057804107, + 2.664999884597242, + 0.3717349596827126, + 2.023207204821542, + 4.832012340829961, + 4.019713584929235, + 2.2620320698268612, + 2.4081223270917347, + 2.3792320159722675, + 1.9266225905158436, + 1.085105883420539 + ], + "mean_band_dot": [ + 8.362911216863717e-06, + 2.207655655865892e-05, + 2.2646067498044422e-05, + 3.211256777717608e-05, + 7.543555866504906e-05, + 3.1522264572458886e-05, + 2.6263951161809013e-05, + 1.280879941987223e-05, + 3.7474966461559234e-05, + 7.846218240956659e-05, + 8.519453760982288e-05, + 8.329713182320118e-05, + 8.766330477101294e-05, + 9.238666168585039e-05, + 7.964869464558433e-05, + 9.174156862172824e-05, + 0.00010365190668153445, + 8.291731659255673e-05, + 8.772217961450222e-05, + 9.432547283267924e-05, + 0.00010294270630311074, + 9.21994996815556e-05, + 8.953784792993245e-05, + 9.780627140116848e-05, + 0.00010209640940956886, + 9.045189403877883e-05, + 9.741334827140234e-05, + 0.00010036427970590011, + 9.989046868668082e-05, + 9.705006905278423e-05, + 0.00010510791573778987, + 9.171266692646897e-05, + 0.00010330190527838567, + 8.547005907075802e-05, + 9.866695493201403e-05, + 9.472228780396106e-05, + 0.00010112459901847614, + 0.00010282318762619978, + 0.00010408317555175019, + 9.884192257914036e-05 + ] + } + ], + "elapsed_s": 7.755878210067749 +} \ No newline at end of file diff --git a/data/exp_wqk_spectral/pythia-410m_wqk_spectral.json b/data/exp_wqk_spectral/pythia-410m_wqk_spectral.json new file mode 100644 index 0000000000000000000000000000000000000000..83f423dd03ac22ac774b096958a07da526b08996 --- /dev/null +++ b/data/exp_wqk_spectral/pythia-410m_wqk_spectral.json @@ -0,0 +1,2631 @@ +{ + "model": "EleutherAI/pythia-410m", + "short_name": "pythia-410m", + "theta": 10000, + "T_train": 2048, + "d_head": 64, + "n_pairs": 32, + "k_dead": 23, + "global_mean_snr": 0.8744793312612019, + "global_min_snr": 0.7784446823209714, + "global_max_snr": 0.9809212291052563, + "layers": [ + { + "layer": 0, + "mean_snr": 0.8813901047194421, + "per_head_snr": [ + 0.8787963779702985, + 0.8423617137100323, + 0.9076139329253423, + 0.9131099727400129, + 0.8925602962295051, + 0.8785762978038703, + 0.9276430131383033, + 0.805419639240085, + 0.941947575924866, + 0.9470376981744225, + 0.9350483256423883, + 0.8723292535808193, + 0.830299369748888, + 0.89711744648364, + 0.8314859098857837, + 0.8008948523128163 + ], + "mean_band_energy": [ + 0.6682215117302346, + 0.5988444907014077, + 0.6977784022379268, + 1.33816765088081, + 0.6663081383003862, + 0.5854269012328666, + 0.8630508985199823, + 1.3145829242384255, + 1.3947611409295013, + 1.4113250232997454, + 1.3999905543209559, + 1.3535653669561867, + 1.3908027482435654, + 1.3538107194387201, + 1.3851371046518521, + 1.3800470537093783, + 1.4021704132594865, + 1.399756752342282, + 1.3954160657198253, + 1.3727214529833347, + 1.3721780437561195, + 1.365073655914047, + 1.3505951707423642, + 1.3870047608887244, + 1.3841821321706054, + 1.37635601126843, + 1.382440559309156, + 1.3841388777335177, + 1.3935587375365228, + 1.359961910040077, + 1.368653836366887, + 1.3879467802685272 + ], + "dft_magnitude": [ + 39.88397578969185, + 3.9180093815280146, + 2.9294750202583835, + 1.5802853154012881, + 0.12921893494702522, + 1.0894529076229953, + 1.6756225596097865, + 1.7460949662118885, + 1.4775339569827055, + 1.0359621717149687, + 0.508048930976683, + 0.2202304356974507, + 0.179493079028351, + 0.39087620720057265, + 0.7032111398395493, + 0.8955076749878981, + 0.8534848741011061 + ], + "mean_band_dot": [ + -2.560590623090775e-05, + -4.6388470877900545e-06, + -2.426727746751567e-06, + -3.817749490053757e-05, + -2.693671412146159e-05, + -9.238854110193984e-06, + -1.927306234961179e-05, + -3.0340313344368042e-05, + -5.021645165470545e-05, + -6.51727301885785e-05, + -4.655271885667389e-05, + -3.7638367871295486e-05, + -4.582685664900055e-05, + -3.7806199458145784e-05, + -4.9914398616124345e-05, + -4.928938744797051e-05, + -4.499941260860396e-05, + -5.224980975526705e-05, + -4.2523093242152754e-05, + -4.9498099770062254e-05, + -3.899492853065567e-05, + -3.953428466729747e-05, + -4.1873026034977556e-05, + -5.1352113359826035e-05, + -4.945167881942325e-05, + -4.163090062547781e-05, + -3.8209503600228345e-05, + -4.5365467315150454e-05, + -3.8719024729516605e-05, + -3.7802954096832764e-05, + -5.0025361105099364e-05, + -3.945690616546926e-05 + ] + }, + { + "layer": 1, + "mean_snr": 0.9022194241595125, + "per_head_snr": [ + 0.8600536228903454, + 0.9040896886593993, + 0.9143128056424027, + 0.9122470657331496, + 0.9507764028687453, + 0.8683261984209641, + 0.9010242402297667, + 0.8851266795443126, + 0.8902167751670043, + 0.8618149217935728, + 0.949312252487469, + 0.8922463058738074, + 0.9182925084763733, + 0.897344600979952, + 0.9544263074484147, + 0.8759004103365207 + ], + "mean_band_energy": [ + 1.0705059316011662, + 1.0317424331911271, + 1.2747378454612832, + 1.8877190670824278, + 1.235448323608754, + 1.0582752112393852, + 1.3397781385589913, + 1.8262015168292978, + 1.851824126167922, + 1.8636889822262228, + 1.9085422959949634, + 1.859792769376285, + 1.8796283822585282, + 1.8641521049807448, + 1.8648729583263197, + 1.8778581150349432, + 1.8777007001533539, + 1.8667531605704033, + 1.8810120064358111, + 1.887356746892097, + 1.8424593447939852, + 1.8812078928234062, + 1.8714553953809192, + 1.8637613670519526, + 1.8501478264419564, + 1.9000969017844094, + 1.87745156694909, + 1.9043109677652463, + 1.883049349455808, + 1.874640417729915, + 1.8510086642191892, + 1.8905304849927527 + ], + "dft_magnitude": [ + 55.69771099537866, + 3.862850344756237, + 2.8680651925129808, + 1.37781299212971, + 0.2658400025630483, + 1.2959586715159999, + 1.8475252655535745, + 1.9961911657208018, + 1.6995641646453703, + 1.0945609519025552, + 0.5939532888705017, + 0.04962471813946183, + 0.12587522372565965, + 0.2667638692208977, + 0.5826189207590863, + 0.8069862411313774, + 0.9784652837625742 + ], + "mean_band_dot": [ + -4.407314804666385e-06, + 4.4159477567973227e-05, + 9.952175304306365e-06, + -7.630452591911308e-05, + 1.3732699414958915e-05, + 2.2439486883740756e-05, + 3.310364371600372e-05, + -6.0860049416078255e-05, + -1.8193645004771497e-05, + -2.4152944433808443e-05, + -1.4951384258665712e-05, + -1.5660253154692327e-05, + -1.5621806767285307e-05, + -1.2053111845489184e-05, + -8.102311880975321e-06, + -2.3283446580535383e-05, + -1.3598428211025748e-05, + -1.008621362075246e-05, + -1.2060855283380079e-05, + -2.5818487756623654e-05, + -2.8015112249590857e-05, + -1.4985750567575451e-05, + -1.7788565259024836e-05, + -1.023136491085097e-05, + -1.6858703190791857e-05, + -1.655209467799068e-05, + -8.215953926082875e-07, + -2.3180491467655884e-05, + -2.014883148149238e-05, + -1.8990555190612213e-05, + -8.32536659345351e-06, + -8.292129109577218e-06 + ] + }, + { + "layer": 2, + "mean_snr": 0.9159249117366062, + "per_head_snr": [ + 0.8950286134459423, + 0.9409796182932284, + 0.9336415144374897, + 0.8904186293453112, + 0.9488267991308942, + 0.9032440633965574, + 0.9638896224914769, + 0.9681135872997237, + 0.9310769448443821, + 0.8743165400904367, + 0.8815784057828787, + 0.8995092803720762, + 0.8692202035407441, + 0.930951893962482, + 0.8994201268464547, + 0.9245827445056198 + ], + "mean_band_energy": [ + 1.224997468064183, + 1.2388115640757476, + 1.2591979879280515, + 1.8308590425861386, + 1.2611037980043567, + 1.1728017866388174, + 1.3387089070128906, + 1.7407418101561176, + 1.9002728213565563, + 1.9022572553493857, + 1.863365423526555, + 1.8716352752086505, + 1.887614096962551, + 1.8816613169434238, + 1.8657565951541142, + 1.9050895890853354, + 1.8810061356973051, + 1.9137774703951833, + 1.895722602810177, + 1.892178253511494, + 1.8860007986200733, + 1.892207615777762, + 1.8895989266218747, + 1.903261570895351, + 1.844800075933727, + 1.8941560409619997, + 1.8668396490149357, + 1.8706829841588961, + 1.861152494866949, + 1.8459607827361069, + 1.880062501164379, + 1.9047783308715776 + ], + "dft_magnitude": [ + 56.26706097209066, + 3.6831081556558476, + 2.4742289241817974, + 1.3216001722643018, + 0.06485117798936378, + 1.0353103182104146, + 1.5248424723379765, + 1.6360777668033164, + 1.1829360593967537, + 0.6408599989044397, + 0.363239793557127, + 0.10586850567111712, + 0.2545925215322969, + 0.49274225705457236, + 0.6508480847366966, + 0.7613903164063567, + 1.054660406613312 + ], + "mean_band_dot": [ + -1.1404941858472739e-05, + 4.4975013224757276e-05, + 4.376344749346117e-05, + 3.371294872067665e-05, + -2.9571025521590855e-05, + 4.877265533309583e-05, + 3.137012902243441e-05, + 2.9980167681742387e-05, + 0.0001297027971531861, + 0.0001330613039272066, + 0.0001290675078280401, + 0.0001395126193415308, + 0.00012784888394890004, + 0.0001410859006227838, + 0.00011039774483379006, + 0.00011947213351959363, + 0.00012627984892787936, + 0.00013086613694213156, + 0.0001385511284297536, + 0.0001406968424362276, + 0.00012468751936012268, + 0.0001377015269099502, + 0.0001278042166177329, + 0.00014523967797686055, + 0.00011974291078331589, + 0.00014496110327399947, + 0.00014736905882273277, + 0.00012111366251588151, + 0.00013038501015216752, + 0.00012210580325699993, + 0.00013304752016551902, + 0.00012347585942507067 + ] + }, + { + "layer": 3, + "mean_snr": 0.9148791254879303, + "per_head_snr": [ + 0.9162995560558217, + 1.0291127093744348, + 0.8468414794908343, + 0.9177583828462782, + 0.8884811769211549, + 0.8892118473997318, + 0.9619173723857117, + 0.8369079450140261, + 0.91314968520748, + 0.9097207974173901, + 0.9115480652075634, + 0.9282388477725152, + 0.9947630866336103, + 0.9021038731954254, + 0.8915425835666724, + 0.9004685993182322 + ], + "mean_band_energy": [ + 1.2997115379047748, + 1.462340494789451, + 1.6370036177349476, + 2.4883014925621723, + 1.3465934104090915, + 1.382896625944395, + 1.7384221417323036, + 2.141065432272339, + 2.3033321731919125, + 2.3203962911518774, + 2.308797615780442, + 2.3236286332042724, + 2.3300637908588486, + 2.3398078473128825, + 2.324536370604659, + 2.3574586272530773, + 2.3103156130467406, + 2.336074426141165, + 2.331857742753333, + 2.3467608613633644, + 2.283011213560652, + 2.3124805251954728, + 2.2930456065841778, + 2.3141168664730687, + 2.3497697136973557, + 2.3233106652613538, + 2.2892191191403723, + 2.339704202412433, + 2.31312212191808, + 2.3522552156582517, + 2.2999211113330063, + 2.3102681461320174 + ], + "dft_magnitude": [ + 69.2095892533783, + 4.617671559845201, + 3.274926472830581, + 1.3935881412559792, + 0.3197095355974948, + 1.6139520687107223, + 2.3945456056442365, + 2.211493177955837, + 1.9188927255136154, + 1.3560730483174552, + 0.8661548689288103, + 0.39130265563932876, + 0.4492116469373686, + 0.7038150598711839, + 1.1714756116102114, + 1.3152533198192526, + 1.6921434528768913 + ], + "mean_band_dot": [ + -1.9102972828477505e-05, + 1.0263705817692426e-05, + 1.1568892489322025e-05, + -5.3095393809599045e-05, + -2.727461040308299e-05, + 1.357762062070833e-05, + -7.956047340940131e-06, + -2.280805290411081e-05, + 3.8910490388843755e-05, + 4.120090625292505e-05, + 4.109522868134263e-05, + 5.0566194090606587e-05, + 3.887871841357082e-05, + 2.775457591042141e-05, + 3.773347293645202e-05, + 4.714201500632953e-05, + 2.7616878114145038e-05, + 2.095293353931993e-05, + 3.795032750986138e-05, + 3.1300093979780286e-05, + 3.0270502406892774e-05, + 4.611770316387265e-05, + 4.097122689472599e-05, + 5.057530688645784e-05, + 4.395991811634303e-05, + 4.3234907877831574e-05, + 3.096454457818254e-05, + 3.531973359827134e-05, + 4.052349171956848e-05, + 5.047118946777118e-05, + 4.098890468640093e-05, + 5.016854419181982e-05 + ] + }, + { + "layer": 4, + "mean_snr": 0.8975187685232682, + "per_head_snr": [ + 0.8991451975913708, + 0.8686958728091064, + 0.88918453819872, + 0.890040949804334, + 0.8284155446792105, + 0.8860074188940426, + 0.8908370862751888, + 0.8851242343784934, + 0.8623881900965678, + 0.9565386891200724, + 0.9223215385240071, + 0.8733226522782308, + 0.8766784601170535, + 0.8651217443774736, + 1.0261089866379032, + 0.9403691925905174 + ], + "mean_band_energy": [ + 1.2423187678096288, + 1.1861149613888307, + 1.263870915373427, + 1.9718681953729176, + 1.175637396419684, + 1.170891954206017, + 1.2630859359203175, + 1.8787703194744205, + 2.0687515913469365, + 2.0367457188533002, + 2.0812799369567196, + 2.0723760334815964, + 2.0561385724719745, + 2.0345345142871913, + 2.0038753219205203, + 2.037081307530621, + 2.0612664616091676, + 2.0126796018230992, + 2.035718401820625, + 2.0701281248395356, + 2.0266458969921377, + 2.041715256252675, + 2.0937844933891903, + 2.0524704846039574, + 2.071179598950545, + 2.032124643087837, + 2.0569474013893845, + 2.037856105179201, + 2.0148346639637724, + 2.074915382125134, + 2.0445567245300627, + 2.0648179083813427 + ], + "dft_magnitude": [ + 60.33498259175177, + 4.700231594471373, + 3.480357328548309, + 1.7714337052415243, + 0.09159530378206608, + 1.414900209007028, + 1.9406121110821215, + 1.9983767130564638, + 1.6006407907306712, + 0.7197457578147695, + 0.48378406184180583, + 0.13978109355481558, + 0.29322795860905865, + 0.3792322026621112, + 0.9001771468052538, + 1.2042121206456766, + 1.2151984300235839 + ], + "mean_band_dot": [ + 3.073296952038618e-05, + 3.746954780581291e-05, + 2.1281178987919702e-05, + 3.2674447993485956e-05, + 6.964136815668098e-06, + 3.9669474688253104e-05, + 9.117467698160908e-06, + 1.819024305405037e-05, + 0.00011477597126940964, + 0.00011003858298863634, + 0.00010234880869575136, + 0.00011254406342686707, + 9.879697631731688e-05, + 0.00011396295076337992, + 9.12789033691297e-05, + 0.00010572280871201656, + 0.00011761464315895864, + 0.00010200954687888952, + 0.00011501833171223552, + 0.00012074559526809026, + 0.00011972620336564432, + 0.00012127469611300512, + 9.329900785814971e-05, + 9.173107878268638e-05, + 0.00011920499309781007, + 0.00012271883633729885, + 0.00012150831628332526, + 9.972589680273813e-05, + 0.00012555532470059916, + 0.00010057753792125368, + 0.00011243964195273293, + 0.0001423867306016291 + ] + }, + { + "layer": 5, + "mean_snr": 0.9809212291052563, + "per_head_snr": [ + 0.9419116440241518, + 0.9292973911287141, + 1.0351607876116227, + 0.958595467948099, + 0.9531305349155891, + 0.9918805167931967, + 1.192878754434385, + 1.10881206303597, + 0.9271941681816717, + 0.9790275927485609, + 0.8954268242153919, + 0.9941545237331442, + 0.963435805527085, + 0.960555107603226, + 0.9556180886731792, + 0.9076603951101123 + ], + "mean_band_energy": [ + 1.2137666181605704, + 1.484773087055088, + 1.4879393230161164, + 1.6406616976589163, + 1.2535191477030292, + 1.4245448386519253, + 1.4200833268140247, + 1.785217527686681, + 1.6447623081649154, + 1.6807726217741692, + 1.6736886966337252, + 1.6609819317123669, + 1.6702255106155028, + 1.67894144747966, + 1.6756236776122344, + 1.6849427596561557, + 1.6633020518872752, + 1.6739299607120328, + 1.6589680613698439, + 1.671672472436421, + 1.6739594361157593, + 1.6558272527295614, + 1.6847540549792999, + 1.6699734636371208, + 1.6784692419452765, + 1.6723633316056339, + 1.6648635851840474, + 1.6631990555973817, + 1.6709062832582193, + 1.6667504535410278, + 1.6473889648407622, + 1.6816358796258553 + ], + "dft_magnitude": [ + 51.778408069860596, + 1.5547439193312949, + 1.1620487789908482, + 0.5693960118123388, + 0.12399214447780972, + 0.5086927649090841, + 0.8181638826798986, + 0.8159329698733266, + 0.6843155452790378, + 0.3904184580363156, + 0.16835598879479474, + 0.32173227632067114, + 0.2897402094593824, + 0.029737636341357093, + 0.4844231469884396, + 0.8651841425983076, + 1.0139674932593934 + ], + "mean_band_dot": [ + 2.020554447312861e-05, + 5.525353797963817e-05, + -2.5541011538621206e-05, + -8.388582728002802e-05, + 4.0615986620196054e-05, + 7.821572694410861e-05, + -2.193018761431631e-05, + -8.215684405854518e-05, + -1.3013440153031297e-05, + -1.3273071317598806e-05, + -1.035793049197764e-05, + -1.6943927619195165e-05, + -1.8996130108916987e-05, + -1.3180346542185362e-05, + -4.912021580594228e-06, + -1.2407759300003818e-05, + -9.60711120256974e-06, + -1.8922222324135873e-05, + -4.35830671108306e-06, + -7.863232781346596e-06, + -1.4593798312034778e-06, + -7.500177218844328e-06, + -3.1724238738206623e-06, + -1.7750290425055937e-05, + -9.916064087178711e-06, + -1.1204447048385191e-05, + -5.18934954385486e-06, + -6.835068134591893e-06, + -9.880090203751024e-06, + -1.0621392334542179e-05, + -9.767081579070691e-06, + -8.058553589762596e-06 + ] + }, + { + "layer": 6, + "mean_snr": 0.8923665388645803, + "per_head_snr": [ + 0.9309054811694172, + 0.8981735011125215, + 0.8323580300793624, + 0.8502971044134805, + 0.8845610573336299, + 0.9341173790072269, + 0.8725250700567024, + 0.8849135836911795, + 0.8553703147759217, + 0.9182029599968892, + 0.8782809955695281, + 0.8476445563792051, + 0.9194089581999422, + 0.9829208465643423, + 0.9175669641865336, + 0.8706178192974009 + ], + "mean_band_energy": [ + 1.0934979801382625, + 1.2451921000732815, + 1.3738146769796313, + 1.7558284422105772, + 1.106874376084984, + 1.2352846816290726, + 1.3516439297767324, + 1.761248248074546, + 1.936982127235094, + 1.9253851322962499, + 1.8955184222800403, + 1.8967842325010142, + 1.9418673946339196, + 1.8869142933293155, + 1.9421757371373793, + 1.946819112032939, + 1.9292411064983375, + 1.9273073693945726, + 1.9005094213233145, + 1.9046426081494836, + 1.9602435132366791, + 1.929711201259006, + 1.9082075445119422, + 1.9693252720532328, + 1.9686676741633562, + 1.9601300600701244, + 1.9326724134692554, + 1.9513957447599592, + 1.9264300051139003, + 1.9367502183088172, + 1.9439067238166068, + 1.9295831347701178 + ], + "dft_magnitude": [ + 57.274554897311745, + 4.118661057577863, + 2.987265460933073, + 1.348209558672302, + 0.12854978029031625, + 1.104590766311107, + 1.6640387331102786, + 1.5582997207918279, + 1.1360498923078621, + 0.7974417123049353, + 0.5194550457815714, + 0.29777883530210114, + 0.12767008830339413, + 0.3301943793749143, + 0.7167611765393908, + 1.014058039090663, + 1.0500488045128726 + ], + "mean_band_dot": [ + -7.2404393307579085e-06, + 1.5885316656749637e-05, + 4.5854362497266266e-05, + 3.032041911410488e-05, + -4.3660597782491095e-05, + 2.5311963838703377e-05, + 2.5905841170015265e-05, + 4.768856047121517e-05, + 0.00015458429675163643, + 0.00016750263858966719, + 0.00014088975717641006, + 0.00014244086491999042, + 0.0001621253004486789, + 0.00013491450749825162, + 0.00016911215323034412, + 0.00016296702551699127, + 0.00015203448583633872, + 0.00015816868346973934, + 0.00015242866061271343, + 0.000155257325218372, + 0.0001550060317754287, + 0.00015375889256574737, + 0.0001666578402819141, + 0.00015063768117329346, + 0.00015830631923563487, + 0.0001582381345315298, + 0.00015696666844178253, + 0.0001402452176364477, + 0.00016969478519968106, + 0.00015003016858372575, + 0.00014082527673053846, + 0.00015406134025397478 + ] + }, + { + "layer": 7, + "mean_snr": 0.9086188681146387, + "per_head_snr": [ + 0.844171100634563, + 0.8329764094117147, + 1.0549533673923546, + 0.9420431742219026, + 0.8556122883855473, + 0.899852793013196, + 0.9266432383662295, + 0.8434383929206248, + 0.829922373060354, + 0.9592930968825699, + 0.9355225009513753, + 0.9003402009719101, + 0.931431643670464, + 0.9062682027376963, + 0.9203633929970887, + 0.9550697142166276 + ], + "mean_band_energy": [ + 0.971201984610804, + 1.1897057915386666, + 1.3750109614884982, + 1.7961062125960603, + 0.9343040303102419, + 1.1504299967021534, + 1.3288358640991929, + 1.7685058694247702, + 1.8773990529913833, + 1.9486074648645761, + 1.904434527353013, + 1.9137964491194577, + 2.0324056865010114, + 1.8990229795514173, + 1.9375984803335715, + 1.9264861609100867, + 1.9241559856533295, + 1.9303991056311576, + 1.959565561507275, + 1.9197721165318677, + 1.9641482738517109, + 1.8950842078666463, + 1.8798048571555555, + 1.9484728712596384, + 1.8955029972714188, + 1.9164570958296476, + 1.915275650803693, + 1.8701532105834122, + 1.9281907232542723, + 1.8197567839213287, + 1.9748590102824632, + 1.8486853593902453 + ], + "dft_magnitude": [ + 56.54413532318857, + 4.549039226850935, + 3.1063070077324446, + 1.4737310871222629, + 0.19887145996171146, + 1.0759546255832113, + 1.8691889913600999, + 1.8659656130389286, + 1.4503313609632302, + 1.0448349011777622, + 0.711516872236715, + 0.27315383108638097, + 0.3648884817637748, + 0.581018085051891, + 0.9651967623751789, + 1.1545747115033513, + 0.9387480282536984 + ], + "mean_band_dot": [ + -3.1889346615798786e-05, + 2.983160757707992e-05, + 2.672028386996317e-05, + 6.870039703699149e-05, + -4.3506545580385136e-05, + 4.223206261144696e-05, + 2.136246749273596e-05, + 9.116538274156483e-05, + 0.0001719637076575964, + 0.0001923875522606977, + 0.00019911751132895006, + 0.00019228815790484077, + 0.0002238900788142928, + 0.00017361897084811062, + 0.00018025594124537747, + 0.0001844944770255097, + 0.00017668614032118057, + 0.00020100457402350003, + 0.0002346519681850623, + 0.00016402375172219763, + 0.00020061009104210825, + 0.0001957267858188061, + 0.00016788995907290882, + 0.0002088241039928107, + 0.00019894772935913352, + 0.00020693509077318595, + 0.00017688450770947384, + 0.00015914557138785312, + 0.0001922913399994286, + 0.0001672937537477992, + 0.00021989332446992194, + 0.00016185601680263062 + ] + }, + { + "layer": 8, + "mean_snr": 0.9082392118240011, + "per_head_snr": [ + 0.9533160531411791, + 0.9566566971705037, + 1.053738019825499, + 0.900538552108297, + 0.9118804010413752, + 0.9124400483086138, + 0.7987455186299974, + 0.794357327041394, + 0.963707130743975, + 0.7977417661002664, + 0.9080439276384937, + 0.9302098145607415, + 0.9503309673955576, + 0.8515437926910877, + 0.9506374731695777, + 0.8979398996174585 + ], + "mean_band_energy": [ + 0.8956407856328843, + 1.1915852121684296, + 1.4519938204753196, + 2.021185408893597, + 0.9075380966711579, + 1.1240379330189423, + 1.42461281300214, + 1.9853514494654947, + 2.048967917579171, + 2.078393225243593, + 2.0539664591784472, + 2.062876031865356, + 2.062245142121138, + 2.0467257852362337, + 2.062543899451507, + 2.093377830560767, + 2.1180531277745454, + 2.0525220772833004, + 2.1527536904275983, + 2.1200405779563196, + 2.1228494588654447, + 2.031975504226019, + 2.0885216129878, + 2.085226435213359, + 2.0911932407687335, + 2.0858206711677933, + 2.073582816064177, + 2.014308472281278, + 1.9890023697061112, + 2.1076942271237114, + 2.0375218587502086, + 2.12850701282742 + ], + "dft_magnitude": [ + 60.810614963988, + 5.262895823514084, + 3.3759159203057907, + 1.8877994938165956, + 0.23672989543536785, + 1.376251689613186, + 2.4794432823381367, + 2.4068287791093423, + 2.1080332499795253, + 1.220183342725073, + 0.8746522939555782, + 0.39959210524957023, + 0.11353370124988449, + 0.4984823868859256, + 0.9421986219849406, + 1.784885271476004, + 1.6486407450752303 + ], + "mean_band_dot": [ + -5.6892579081591066e-05, + 4.1481204675619665e-06, + -2.0701954781543463e-05, + 3.557990586955384e-05, + -5.258097306182208e-05, + 7.54965202531821e-06, + -1.3227386375547212e-05, + 5.9504635601115297e-05, + 0.00015489646543187519, + 0.00014752895799574617, + 0.00011912561029703284, + 0.00014986047415277426, + 0.00015929149299154233, + 0.0001343948720489152, + 0.00012987331811586955, + 0.00015024338574676221, + 0.00016297068941639736, + 0.00012895309562566126, + 0.00016611771448538093, + 0.00014263542942671847, + 0.0001552682853684928, + 0.0001399751069470767, + 0.00013173140098388103, + 0.00016997259075424154, + 0.0001507920267300733, + 0.00017710136521031927, + 0.00015067851043681912, + 0.00013786079392730244, + 0.00015975371752574574, + 0.00012894166363253134, + 0.000157352795710608, + 0.00014638633149388625 + ] + }, + { + "layer": 9, + "mean_snr": 0.9067223659694099, + "per_head_snr": [ + 0.8086725150435055, + 0.9957201994805747, + 0.9104255628488804, + 0.8396153948588791, + 0.8454356013438369, + 1.1414994057491432, + 0.8324599696739502, + 0.9273015538730125, + 0.9224384093923079, + 0.814298131538081, + 0.8983924687120948, + 0.9012687832935566, + 0.9239616525807154, + 0.9958691645202877, + 0.8794247396181011, + 0.8707743029836326 + ], + "mean_band_energy": [ + 0.7941411943462966, + 1.042423629450987, + 1.3495807437127714, + 1.8287836076938202, + 0.840726127111856, + 1.0623022029231242, + 1.3018717085071594, + 1.824645036563711, + 1.8645695420941224, + 1.8447923679149483, + 1.8911340764110376, + 1.8828214495165958, + 1.870108368687482, + 1.8311487198003755, + 1.9002990682480365, + 1.862075340563683, + 1.8354723951598018, + 1.8590470008172775, + 1.9271027325198995, + 1.8800309599880245, + 1.8221559467039548, + 1.89324108377431, + 1.8611686069411375, + 1.8676416102808036, + 1.9022448922658823, + 1.858214308903651, + 1.8867432254574465, + 1.821930470084499, + 1.8746179357328314, + 1.886500734046389, + 1.8726249747005088, + 1.8717358471649568 + ], + "dft_magnitude": [ + 54.91189590808738, + 4.477405150348568, + 3.2424712544569148, + 1.629733912966387, + 0.07956470318044748, + 1.3454118715166135, + 2.2211141226405964, + 2.148684287860911, + 1.9615253323778057, + 1.1534768591435927, + 0.813996590661856, + 0.3479488444333756, + 0.17834621136745196, + 0.40232351929378607, + 1.0665518110994654, + 1.3328072069443124, + 1.322772830886933 + ], + "mean_band_dot": [ + -8.254147508068854e-05, + 1.104180901734253e-05, + 1.854241504872789e-05, + 4.1538343452884874e-05, + -6.082019001496519e-05, + -9.699156265696729e-06, + -5.103868829792191e-06, + 3.245873631385621e-05, + 0.0001717318667147083, + 0.00016253811691058218, + 0.0001645646939323342, + 0.0001560629075356701, + 0.00015807287110192192, + 0.00016063722137005243, + 0.00018729189707755722, + 0.00016846736468778545, + 0.00017961371395358583, + 0.0001662623192260071, + 0.00017066856855763035, + 0.00017339347130018723, + 0.00013875921933959035, + 0.00013342574769126259, + 0.00014668981657450786, + 0.00017302552362252754, + 0.00017726094722547714, + 0.00013730470038808562, + 0.00017406771939931787, + 0.00016060564314557269, + 0.00014942021692831986, + 0.00017063682139450975, + 0.00015511202286688786, + 0.00015669193271605764 + ] + }, + { + "layer": 10, + "mean_snr": 0.8922940906784538, + "per_head_snr": [ + 0.8056820259875137, + 1.0298346701252556, + 0.8984478340372051, + 0.7613706492096853, + 0.866221409301464, + 0.9447883328174879, + 0.9125487002701521, + 0.8631367628838984, + 0.9136535547610677, + 0.7952469617080334, + 0.9633023557183246, + 0.945339363347901, + 0.8651449655322263, + 0.86630687400587, + 0.978266959559811, + 0.8674140315893634 + ], + "mean_band_energy": [ + 0.8845807009185569, + 1.2021363178527351, + 1.573524803619872, + 2.2248105647976817, + 0.931282321658943, + 1.2250868830407868, + 1.585287452586226, + 2.1550551179090554, + 2.172336919674356, + 2.184411714261593, + 2.193181182597203, + 2.151016160844443, + 2.165577329510435, + 2.1687458463330493, + 2.1649359021257553, + 2.173725993193872, + 2.2105379577023845, + 2.1494323311951193, + 2.20338909527542, + 2.1895552940976097, + 2.19712725436165, + 2.1773837424715268, + 2.1528549073684955, + 2.1941067288944014, + 2.160783902912974, + 2.230549149590952, + 2.216216614602396, + 2.2207526156718354, + 2.1720894214580744, + 2.220319776587301, + 2.225502200942783, + 2.223051397268951 + ], + "dft_magnitude": [ + 64.29934760132643, + 5.107243689932387, + 3.8458300939146106, + 2.0220339816303716, + 0.10832600066375189, + 1.5973626312621836, + 2.6170758268399137, + 2.8370379658689906, + 2.4320249160833867, + 1.6754256768531155, + 0.9699259887003824, + 0.377327667037814, + 0.047713055126466085, + 0.6723939711215056, + 1.0652126163892217, + 1.7459618224871127, + 1.8809316666953855 + ], + "mean_band_dot": [ + -8.834047935124545e-05, + -1.7171157026041328e-05, + -2.5804859177469552e-05, + 7.495560549841684e-05, + -6.361938795862443e-05, + -1.8971892885133457e-05, + -2.4789488350052125e-05, + 9.692898379398684e-05, + 0.00019115938910374553, + 0.00021451455663168417, + 0.00021071687262974592, + 0.00020380764806304796, + 0.00018630887086601433, + 0.00021428888067021035, + 0.00018663661819573463, + 0.00019232962699788914, + 0.00018739132343625897, + 0.00017314701409532063, + 0.0002006439046340347, + 0.0002212636363196907, + 0.00021505584163605818, + 0.00021901372679167252, + 0.0001935505690937589, + 0.00022072780768667144, + 0.0002202087231353289, + 0.00021639964433006753, + 0.00021065142897214173, + 0.000204616788948897, + 0.00019599306665440963, + 0.0002025745327216555, + 0.000229586346506494, + 0.00021082145710238365 + ] + }, + { + "layer": 11, + "mean_snr": 0.9072260454164791, + "per_head_snr": [ + 0.9233065049290191, + 0.8204610711848839, + 0.8258239132983024, + 0.9342720044493101, + 0.9879212039509236, + 0.8838815277509209, + 0.8267475937605526, + 0.8852841560447872, + 0.9827920836875085, + 1.0162986107354952, + 0.9170394655112416, + 0.90441912825634, + 0.9490728185743578, + 0.9693820924967486, + 0.8194376475046978, + 0.8694769045285768 + ], + "mean_band_energy": [ + 0.8280037894543704, + 1.3337580452360387, + 1.769824150874678, + 2.2709372802691483, + 0.8579986855708432, + 1.308284295857137, + 1.6898329086966402, + 2.403750127435745, + 2.321974234483786, + 2.328198397351528, + 2.2957776341194265, + 2.3419732519334087, + 2.3294307590355245, + 2.2879675980814405, + 2.292375141869078, + 2.308390010191909, + 2.302428962330848, + 2.2913172775647226, + 2.3187064475317305, + 2.3282367434465936, + 2.332143539963953, + 2.3692826964530074, + 2.3095871456347004, + 2.3048508779787493, + 2.271681033660048, + 2.3230909463731173, + 2.309903543688013, + 2.287452801252619, + 2.3239590860562878, + 2.318902960666371, + 2.2969921635199224, + 2.2899442505392775 + ], + "dft_magnitude": [ + 67.94695678712067, + 5.564481656545513, + 4.076347220029625, + 2.2542995830552326, + 0.08265160779558497, + 1.7885001694542257, + 2.787509836520175, + 3.2275500720683126, + 2.6157400838250195, + 1.7224289386649694, + 1.1859021417997455, + 0.5862808731603866, + 0.23734834333946542, + 0.5614968111668167, + 1.231413944556074, + 1.8640059457265574, + 2.2457183341409603 + ], + "mean_band_dot": [ + -5.100572391825153e-05, + -8.86706689584571e-06, + 3.823544443548599e-05, + 4.314814594863492e-05, + -3.4628137562719985e-05, + -9.270698541286038e-06, + 3.173909254883256e-05, + 3.9610188707683847e-05, + 0.00017062802697864754, + 0.00017664683568341388, + 0.00017886082913776136, + 0.00018466152388896262, + 0.00017782528158249988, + 0.00015243486444660448, + 0.00017507042858255772, + 0.00015519984606271464, + 0.00015815699160270924, + 0.00016229133836986875, + 0.0001770425574818546, + 0.00017789599115758392, + 0.00019703366876910877, + 0.00018953918745978626, + 0.0001548458833440236, + 0.000176308453021079, + 0.00014059611226002744, + 0.0001650724071851073, + 0.00019455338230045527, + 0.0001610618291749688, + 0.000170084328374287, + 0.0001899351175893571, + 0.00017300428629596354, + 0.00017633923731352752 + ] + }, + { + "layer": 12, + "mean_snr": 0.927075452714663, + "per_head_snr": [ + 0.9616249905027184, + 0.9233446424647437, + 1.0463193108733782, + 0.9163500182567256, + 0.9571572314151486, + 0.9420597402529423, + 0.9148531867292949, + 0.8894875767253227, + 0.9091998458606104, + 0.8592053523135931, + 0.9296622524146811, + 0.8549223273153188, + 0.9649497943934341, + 0.8980643096172801, + 0.9468900666836808, + 0.9191165976157333 + ], + "mean_band_energy": [ + 1.2007849233948715, + 1.4809739613521282, + 1.6665278373064307, + 2.1819462224594464, + 1.156650019330093, + 1.5370557947198016, + 1.7260466329657724, + 2.112698343956369, + 2.150640568271469, + 2.1992756622822744, + 2.1698038143655785, + 2.1446823342105334, + 2.140629968128799, + 2.188027385152451, + 2.1352798922308383, + 2.083082720687253, + 2.1505326614120106, + 2.1349253746109085, + 2.199867588653989, + 2.162406431604616, + 2.192180415747523, + 2.1346669618915888, + 2.1336370745099673, + 2.1342894188407016, + 2.1322642883354535, + 2.162241968368847, + 2.09887169364045, + 2.1497853229515034, + 2.1609666308960973, + 2.166752308624214, + 2.1524427492920033, + 2.1447619850932735 + ], + "dft_magnitude": [ + 64.68469895528726, + 3.7415546184123842, + 2.7509809368677494, + 1.638237273102712, + 0.1805398467284457, + 1.046713898745315, + 1.920654137885363, + 1.7824721181369423, + 1.492370084741914, + 1.3596237402915643, + 0.7947650875265889, + 0.3728832778863899, + 0.1638220401147751, + 0.6657438119406187, + 0.8340700156706709, + 1.546502181135995, + 1.5504454383245623 + ], + "mean_band_dot": [ + -2.6695556698541623e-06, + 1.870019826810676e-05, + 1.3393000074302108e-05, + 1.4666967388166086e-05, + 7.41362651979216e-06, + 2.8472083045016916e-05, + 1.9177726471752976e-05, + 5.575233558374748e-06, + 0.000125919051697565, + 0.0001510007951424086, + 0.00012569417697250174, + 0.00015524364852126382, + 0.0001228596797773207, + 0.00013269212207944747, + 0.00013627512339553505, + 0.00013382012161855528, + 0.00014964891806812375, + 0.0001388810394473694, + 0.00014210497340627626, + 0.00014172408032209205, + 0.00014406785942355782, + 0.00013631005822389852, + 0.00011484225524327485, + 0.0001439978674966369, + 0.00014593836999665655, + 0.0001356672677275128, + 0.00013277669506805978, + 0.00014866356620757415, + 0.00012176304835520568, + 0.0001333790444277838, + 0.0001176465888192979, + 0.0001495644768283455 + ] + }, + { + "layer": 13, + "mean_snr": 0.8962597948276194, + "per_head_snr": [ + 0.8854073472143543, + 0.9336462872243937, + 0.9505975650422411, + 0.8795410827212289, + 0.8669478859507335, + 0.9567457713099993, + 0.8947525080462, + 0.9177183948629194, + 0.8846163597854454, + 0.9305439626503279, + 0.9306770276312187, + 0.8945336147208636, + 0.7727436674361463, + 0.9035932965717124, + 0.8317601075207949, + 0.9063318385533303 + ], + "mean_band_energy": [ + 1.1908060644281973, + 1.5972173952003974, + 1.5496008462060304, + 2.30353638012373, + 1.1508663589487576, + 1.502903270603802, + 1.7021255436130582, + 2.2973060934091416, + 2.45446847678922, + 2.419279087448924, + 2.420555862359448, + 2.4977767650263427, + 2.3998968488442713, + 2.4264316372905705, + 2.437362424595727, + 2.4763398109432444, + 2.4453556415231006, + 2.4956989278556865, + 2.446403652242993, + 2.406979926273676, + 2.3606794991615745, + 2.437475997195264, + 2.4350528748676368, + 2.445265446229258, + 2.3960867954343312, + 2.460122118493529, + 2.5227872276303884, + 2.4438527213642027, + 2.418697776304991, + 2.411931782524843, + 2.3920871886641715, + 2.4231259097650995 + ], + "dft_magnitude": [ + 71.76807635136161, + 5.689901811363262, + 4.236046277377042, + 1.9671773710431368, + 0.3091942185168338, + 1.41996810805945, + 1.9134583864850891, + 2.356561508423537, + 1.8887579125718628, + 0.9322535401153493, + 0.6132475950956433, + 0.42567748351526513, + 0.1733971776560745, + 0.9380745049165244, + 1.506532712185552, + 1.7845023386639587, + 2.3224101881338157 + ], + "mean_band_dot": [ + 1.7465357956325533e-05, + 2.0505050514429968e-05, + 2.1875001635862645e-05, + -2.5915037440427113e-06, + -3.403856112527137e-07, + 4.3165196359495894e-05, + -1.1057751009957428e-05, + 2.1906311076236307e-06, + 0.00013414599129646376, + 0.00011147191631266651, + 0.00011014950814569602, + 0.0001238480974734557, + 0.0001144210781944821, + 9.925029223722959e-05, + 0.00012235003032401437, + 9.559890219179579e-05, + 0.00013863132778624276, + 0.00011908728686194081, + 0.0001152003011668512, + 0.0001280352284993569, + 0.00014288599538758717, + 0.00013325436668765178, + 0.0001243168934337291, + 0.00011685688139095873, + 0.00011556529977951868, + 0.00011704444511906331, + 0.00012704757978099224, + 0.00012333546465015388, + 9.540708461486247e-05, + 0.0001175319580966061, + 8.148739092916912e-05, + 0.00010932656209661218 + ] + }, + { + "layer": 14, + "mean_snr": 0.8758780610460346, + "per_head_snr": [ + 0.825568446297499, + 0.9096203942494147, + 0.8868419434663947, + 0.7553923409429668, + 0.810844207337527, + 0.9461499237971234, + 0.9013183988465197, + 0.9065389807669997, + 0.891212168672307, + 0.8683740660340931, + 0.8947215946059622, + 0.9940549596112149, + 0.9436447661875362, + 0.8870030540255479, + 0.8867079106081989, + 0.7060558212872479 + ], + "mean_band_energy": [ + 1.7144051754523542, + 1.7153985290932383, + 1.8961843836884835, + 2.512430561426245, + 1.6230306491214879, + 1.8456450868197427, + 1.9969146072110182, + 2.6205140366618203, + 3.496916903027225, + 3.4040052651385357, + 3.761656710987891, + 3.619588507251734, + 3.4629214310957064, + 3.7224252207456026, + 3.389564734083401, + 3.6435388652047394, + 3.9283208443011883, + 3.4106808429420328, + 3.5625868837724584, + 3.50001036228033, + 3.172619768376614, + 3.4322764654353906, + 3.0390822250484337, + 3.699237435230155, + 3.2943182667174025, + 3.580777107735539, + 3.5338585799168722, + 3.776323039213195, + 3.5575196865381145, + 3.421682280645962, + 3.580837165050796, + 3.5931995793869502 + ], + "dft_magnitude": [ + 100.50847119960066, + 10.58093946078601, + 9.128114241102752, + 3.655776605889795, + 0.6569773092184112, + 2.58370784618717, + 2.683857721335639, + 2.6266625161967823, + 2.4849817189324237, + 1.9338790317666241, + 0.9704318091484665, + 1.1503665740203224, + 1.2992761955276901, + 2.2792743106148, + 0.9160272285209963, + 1.6467897276709764, + 2.4869951708217712 + ], + "mean_band_dot": [ + 0.00012975596507658338, + 0.00010622700983731193, + 7.7203313338714e-05, + 9.718001325609293e-05, + 0.00012484307653437554, + 0.00011423095634199854, + 0.00011354257287621294, + 0.00011735930081613333, + 0.00027045557141036625, + 0.0002833278820162377, + 0.00022733125436502633, + 0.0002618748835629958, + 0.00027988050271687825, + 0.000266008499693271, + 0.0002523668520097999, + 0.0002613670617392927, + 0.00029441477022373874, + 0.00026832472406113084, + 0.00026674088803702034, + 0.0002686539053229353, + 0.0002794946842641366, + 0.00029156175128264294, + 0.00030031883625269984, + 0.00023242778490839555, + 0.0002682876290691638, + 0.0002787400071611046, + 0.00023614877568434167, + 0.00028345663213258376, + 0.000268158885774028, + 0.0003009671322615759, + 0.0002544812831501986, + 0.000277859277503012 + ] + }, + { + "layer": 15, + "mean_snr": 0.8923873108234937, + "per_head_snr": [ + 0.8828450045587827, + 0.8869901367504761, + 0.9016253904968258, + 0.8988556021886862, + 0.8449241640031647, + 0.847906053906941, + 0.9021107449552868, + 0.795727540167221, + 0.7979423798104429, + 0.8881304946367222, + 0.9059025202149236, + 0.9802373396627199, + 0.9278469084111503, + 0.9593826950477241, + 0.9371359592588449, + 0.9206340391059864 + ], + "mean_band_energy": [ + 1.2356191603270137, + 1.5474697475865453, + 1.678566368986838, + 2.063772308899156, + 1.2024761631943333, + 1.50923865933009, + 1.660215248516021, + 2.1526535845511754, + 2.5886501993021405, + 2.5267775100084817, + 2.5991378617009753, + 2.454024623868013, + 2.4553573418420234, + 2.5548913289599717, + 2.6482572534763813, + 2.508509349910267, + 2.6190731609254554, + 2.5326736447625473, + 2.562409899904231, + 2.570379345602956, + 2.694465005647655, + 2.640881916912786, + 2.633814656468007, + 2.557243367557576, + 2.5676280309426636, + 2.46552714974281, + 2.609101924914066, + 2.5070549129371997, + 2.523962979828191, + 2.5328837960009443, + 2.643196990909285, + 2.6580012582060517 + ], + "dft_magnitude": [ + 74.70391475172184, + 6.907304777152556, + 4.414764163242212, + 2.761480351525241, + 0.5012640680395629, + 1.5968797917239748, + 2.3307488282929545, + 2.102396070697122, + 1.6325713637393349, + 1.4455908539671185, + 0.752282639652418, + 0.8475918641964195, + 0.187167126357816, + 0.7505071035187384, + 1.3425664440697587, + 1.7100655655022146, + 0.8600502579512934 + ], + "mean_band_dot": [ + 3.2463755317735377e-05, + 4.871806766004738e-06, + 1.6448951811298684e-05, + -8.142622789364395e-06, + 7.332955078709347e-06, + 3.868544311558253e-05, + 1.580676287460392e-05, + 2.1248975713206164e-05, + 0.0001704694353747982, + 0.0001732947029040588, + 0.00019640908271867374, + 0.00019886417794623412, + 0.00017306853749232687, + 0.0002026202312777059, + 0.00017137440647729818, + 0.00018955597784042766, + 0.0001967291361779644, + 0.00019435129200928714, + 0.000177590250814319, + 0.00017657075818533485, + 0.0001730138135371817, + 0.00019425420850893715, + 0.00019948541469716474, + 0.00019458687120277318, + 0.00019446042796289476, + 0.00019538226473514442, + 0.00020270148965551016, + 0.00018860530673237008, + 0.00019935261889258982, + 0.00017575224399024592, + 0.00016446199776964932, + 0.00018662523746115767 + ] + }, + { + "layer": 16, + "mean_snr": 0.8889395519740022, + "per_head_snr": [ + 0.9654709149979019, + 0.8580243445566632, + 0.9650197585735223, + 0.8567234841187472, + 0.8327672404972799, + 0.862460235199133, + 0.8878551768739432, + 0.8493879464195049, + 0.8778088038764312, + 0.9115774698135439, + 0.9557672921915898, + 0.9042098506050111, + 0.8957873467864362, + 0.8623356663123587, + 0.835488265054063, + 0.9023490357079056 + ], + "mean_band_energy": [ + 1.0471391666188683, + 1.337514058473064, + 1.478198403647319, + 2.212842742506936, + 1.0430309322566667, + 1.236589450252651, + 1.4872113706886316, + 2.2868758981998143, + 2.3134318663558284, + 2.293160722395335, + 2.3344697120671647, + 2.315313300347351, + 2.3384261650457425, + 2.3115822569303432, + 2.365687073543434, + 2.300990980045472, + 2.3288212642397905, + 2.297474453677526, + 2.277103390654469, + 2.244128687987148, + 2.3211061591992284, + 2.3676030339078435, + 2.295561441411607, + 2.3030162371010787, + 2.254501315208876, + 2.296717194501416, + 2.3107477915818038, + 2.3044226650434103, + 2.2851517984371457, + 2.3194611060970374, + 2.346377769600254, + 2.3049669932060723 + ], + "dft_magnitude": [ + 67.55962540122934, + 5.733307678791377, + 4.361721024659454, + 2.168701977624596, + 0.1888198399424982, + 1.7254548237165572, + 2.423167529067251, + 2.856247795153745, + 2.0527553952388375, + 1.315761170695915, + 0.4195079122007337, + 0.23293903953221218, + 0.12076685127157019, + 0.6104195034023732, + 1.2448717044525681, + 1.83048324326765, + 1.9056941601156723 + ], + "mean_band_dot": [ + 2.706440702127111e-05, + 3.313155048090266e-07, + -3.252025649658208e-06, + 8.107003714030725e-05, + 1.055354044865453e-05, + 6.991766156261292e-06, + 2.9649226149786045e-06, + 7.471291360161558e-05, + 0.0002668150600300123, + 0.00024908540228807396, + 0.000265477868367725, + 0.0002588628165085538, + 0.0002692231478249596, + 0.0002480542018474807, + 0.0002756742303517967, + 0.0002605911545288109, + 0.000266039037057908, + 0.00025638234637881396, + 0.00026348113453877886, + 0.0002382347433922405, + 0.00026558614661098545, + 0.00027374501939902984, + 0.00026575103129289346, + 0.0002761407893103751, + 0.0002549367512756362, + 0.00025649894666912587, + 0.0002528930410221619, + 0.00026001116293628, + 0.00027226161864746246, + 0.00026630013843487177, + 0.0002618273449570552, + 0.00026144454920995486 + ] + }, + { + "layer": 17, + "mean_snr": 0.8639404125965056, + "per_head_snr": [ + 0.8505475909878651, + 0.804193035000964, + 0.8712162353478976, + 0.759664567095345, + 0.9915111496770107, + 0.9916830354978045, + 0.8427500404656825, + 0.780341331942656, + 0.866329847000895, + 0.8193896158567286, + 0.8707250928179645, + 0.8499566541775796, + 0.9045024065764662, + 0.9340651870987241, + 0.8153213747913556, + 0.8708494372091498 + ], + "mean_band_energy": [ + 1.4204396860407316, + 1.6022619073613045, + 1.874606459231552, + 2.4532574610253466, + 1.5117460713408502, + 1.723459555128502, + 1.8846202554840543, + 2.3774758229507196, + 3.9410729573355487, + 4.272913143440479, + 4.029019189264986, + 4.475567516064092, + 3.9975757964485545, + 3.9453776860798397, + 3.7816829916656696, + 4.038464452976675, + 3.9416733801104806, + 4.0417743761107605, + 4.134230661478006, + 4.146339360121146, + 4.184082591747191, + 3.9709956411098783, + 4.237517379093744, + 4.2569240124878664, + 3.7958418438690256, + 3.8422583262063377, + 4.181027210336639, + 3.8656211995859957, + 3.9916864168117456, + 3.8702586400666865, + 3.7819251460522807, + 3.96283563770967 + ], + "dft_magnitude": [ + 111.53453277473636, + 16.110172402543455, + 10.605815492331821, + 5.922877968920664, + 1.1167283432532722, + 3.979798734004163, + 3.5126938885210253, + 3.233412256496897, + 2.564888300416528, + 1.8738438022244037, + 2.0785060747322897, + 2.406252568808695, + 0.5644479405751591, + 2.1319314753860272, + 2.044040441051765, + 1.5743389752123915, + 2.1570367021142403 + ], + "mean_band_dot": [ + 7.129852571097217e-05, + 4.58931097000459e-05, + 4.166085547296916e-05, + 4.433135359249718e-05, + 9.233678864006833e-05, + 3.902813676859296e-05, + 5.1989671248975355e-05, + 7.274937485135524e-05, + 0.00026822464752740416, + 0.0002604723895274219, + 0.00021655943135101552, + 0.000267845735947958, + 0.0002462655800172797, + 0.0002658164700051202, + 0.00024411335562035674, + 0.0002738924342793325, + 0.00026160671177422046, + 0.000250237008572185, + 0.00026698777355704806, + 0.00025996364047387033, + 0.000251123294503941, + 0.00025014801667566644, + 0.0002876570314356286, + 0.00026892216891383214, + 0.000267557508777827, + 0.0002572866264927143, + 0.00025083009944637524, + 0.0002544003746152157, + 0.00026007102633229806, + 0.0002809728343891038, + 0.00024396557114414463, + 0.0002763382821058258 + ] + }, + { + "layer": 18, + "mean_snr": 0.8025732967546853, + "per_head_snr": [ + 0.7596840495473977, + 0.7362906171608185, + 0.8173804911626837, + 0.6371557071633842, + 0.8131938700927023, + 0.919945854342129, + 0.8997774295121012, + 0.9147903289269246, + 0.8268585931689613, + 0.9051560632691518, + 0.8138827685519745, + 0.8353090374533819, + 0.7466854889082585, + 0.8565111295524924, + 0.7282103756955313, + 0.6303409435670702 + ], + "mean_band_energy": [ + 1.5566057501375365, + 1.928285200105817, + 2.2186042240291446, + 3.3676234187531175, + 1.5587461164875518, + 1.8270714558138228, + 2.242800465203275, + 3.473749104809788, + 6.260758347499557, + 6.128817340210244, + 6.365561113394838, + 6.89193569305731, + 6.497195457140329, + 6.280432431645705, + 6.185558036070143, + 6.07770051863035, + 5.789220306954962, + 6.003625445711163, + 6.208722929708609, + 6.3897920650479625, + 6.17651460638563, + 6.358998082162181, + 6.1942832847315765, + 6.072046764288758, + 6.6454406658120755, + 5.852046121125036, + 5.6662584162934415, + 6.2712588069701525, + 6.835246452574428, + 6.6253834081824525, + 6.533628285249801, + 6.930659336776669 + ], + "dft_magnitude": [ + 169.41456965096341, + 27.131601128163418, + 22.155147797070875, + 12.410428323834848, + 2.293460611961114, + 6.922029466944504, + 8.392401062697996, + 5.28366938088191, + 4.479875183927394, + 3.3384871612031457, + 5.999917180214397, + 2.9960973677865628, + 0.30305849849076955, + 2.9739616078225364, + 5.104179688983737, + 3.598562222212365, + 3.544280735617633 + ], + "mean_band_dot": [ + 0.0001104062123431504, + 4.6635394767235994e-05, + 3.1704523905773385e-05, + 0.00016036864724355837, + 0.00012489705824236808, + 4.161083285225686e-05, + 3.576471839039641e-05, + 0.00013683357644822536, + 0.0003815543436758162, + 0.00038613754509242426, + 0.0003504437645460712, + 0.0003653577969089383, + 0.0003518097119012964, + 0.00031790662887942744, + 0.0003886565173161216, + 0.0003154792821078445, + 0.00034400571922788004, + 0.00034382771718810545, + 0.0003261764077251428, + 0.00031783612257640925, + 0.00036186547004035674, + 0.00036326810277387267, + 0.00036348402136354707, + 0.0003271526791195356, + 0.00037347535817389144, + 0.0003473957358437474, + 0.000315259605031315, + 0.0003259228533352143, + 0.00034897819750767667, + 0.0003433698764183646, + 0.00032337885568267666, + 0.0003630079390859464 + ] + }, + { + "layer": 19, + "mean_snr": 0.7882553426965626, + "per_head_snr": [ + 0.79999274980639, + 0.8626557920298195, + 0.5593421095160993, + 0.7708849293940346, + 0.7234131207598538, + 0.8961426036393784, + 0.6323799564404791, + 0.8275209926761792, + 0.8495717722014562, + 0.8469523702847661, + 0.8092734414011693, + 0.7247815598263234, + 0.8273905412753462, + 0.8584888859392054, + 0.8159805175643094, + 0.8073141403901917 + ], + "mean_band_energy": [ + 1.7068761113166429, + 2.030167418808123, + 2.306777239522666, + 3.1513445118129084, + 1.7114571223244, + 2.0053587588648116, + 2.3532212301984705, + 3.3497209446875083, + 7.374018239162038, + 7.293937109486759, + 6.995412416718809, + 7.035737256900919, + 8.155173136960396, + 8.126033905246528, + 7.379174675227812, + 6.964055216115649, + 7.938884341037165, + 6.2986127170887425, + 7.568526947351819, + 6.694072648652222, + 7.104560481819919, + 5.51562633746761, + 7.350424548068988, + 7.101817234048385, + 7.181712208281338, + 7.771211183940035, + 7.370970957442621, + 7.519830109325769, + 8.329642221838041, + 7.921024693709667, + 6.7811043643773745, + 7.193439904464267 + ], + "dft_magnitude": [ + 193.5799261922684, + 33.62466804852113, + 30.864913221539584, + 9.559623357080188, + 1.0807105355549678, + 6.519881173044439, + 10.104417841966345, + 6.080582434863728, + 2.4789703838631154, + 4.72969312739457, + 5.927367592460742, + 5.413902882643176, + 1.2968349538210664, + 4.284184688626821, + 4.229846389637783, + 7.712342642392021, + 1.6359462910286027 + ], + "mean_band_dot": [ + 0.0001366470941661646, + 8.17830384107765e-05, + 6.801760235930487e-05, + 0.00011632615269263624, + 0.00013752930695432042, + 5.1882016236959316e-05, + 5.31856894667726e-05, + 7.086448454174388e-05, + 0.00036645496356868534, + 0.00035091354402538855, + 0.0003796372043325391, + 0.00038799273534095846, + 0.00031635797495255247, + 0.00035502450634794513, + 0.0003600401510084339, + 0.0003646286095317919, + 0.00032489425075254985, + 0.00035651334928843426, + 0.00033677106966933934, + 0.00033854189723570016, + 0.0003745004887605319, + 0.00036523194239634904, + 0.00035708096129383193, + 0.0003794577278313227, + 0.0003630802129919175, + 0.0003118558493042656, + 0.0003519425381455221, + 0.0003318326657790749, + 0.0002970893497149518, + 0.00036598863698600326, + 0.00036842692861682735, + 0.00033995083686022554 + ] + }, + { + "layer": 20, + "mean_snr": 0.7949488955403285, + "per_head_snr": [ + 0.701917531169913, + 0.733713987908543, + 0.9567648566010388, + 0.7696657724724731, + 0.6782792020979744, + 0.6459601994224485, + 0.9385359786540753, + 0.7799065968670441, + 0.7280500432814534, + 0.8601831458525204, + 0.805534229679056, + 0.8034217438053167, + 0.7753823225720221, + 0.8625632650899241, + 0.8689860911596428, + 0.8103173620118096 + ], + "mean_band_energy": [ + 2.9816744530482207, + 3.355395358953717, + 3.644555883630826, + 5.238951514086555, + 3.1476970479735282, + 3.2591161177458203, + 3.7988984492454607, + 5.165824808688306, + 12.787903442887613, + 12.204127817150754, + 12.334757492042273, + 13.165282968152761, + 14.27005761889738, + 14.01800904230307, + 12.741459691081502, + 12.607254668338737, + 12.815709552487148, + 12.102143940008837, + 13.709590532633332, + 13.026834996795024, + 13.15196767530693, + 13.081063387592035, + 13.601218912529003, + 12.193056874191509, + 13.92425087576202, + 12.996194333678702, + 13.232353069598012, + 12.744175296556145, + 12.394565118363346, + 12.781484229851358, + 12.423773892044066, + 13.015647532222452 + ], + "dft_magnitude": [ + 341.91499659384647, + 67.91041927031603, + 46.721580957499555, + 19.808002183413823, + 3.1036033306313136, + 9.667234688357096, + 17.39909410934808, + 14.592845712397363, + 3.359518748272475, + 6.239658007538617, + 11.498753320807236, + 4.606307945004996, + 2.4903917088620253, + 6.133471109762618, + 11.11561209119447, + 8.655344497044561, + 0.005870821214870148 + ], + "mean_band_dot": [ + 0.0003035891697322768, + 0.0002213955954175617, + 0.0001673575485483525, + 0.00020204723819006176, + 0.00035727179545119725, + 0.00021365562895425683, + 0.00017823417982754108, + 0.0002677780902331506, + 0.0005953826312179444, + 0.0005567435318880598, + 0.0005856445978338343, + 0.0005513293490366777, + 0.0005373869307732093, + 0.0005358008138500736, + 0.0005117300097481348, + 0.0005045917660027044, + 0.0004797121991941822, + 0.00048304301390089677, + 0.0005542499805386569, + 0.00048781694516719654, + 0.0005841009401592601, + 0.00048668419776731753, + 0.0005497989147897897, + 0.00055176008527269, + 0.0005374443148866703, + 0.0005585762382906978, + 0.0005328936154000985, + 0.0005749275480866345, + 0.0005846019430464366, + 0.0005012631704630621, + 0.0005268296304166142, + 0.0005728479805497955 + ] + }, + { + "layer": 21, + "mean_snr": 0.7784446823209714, + "per_head_snr": [ + 0.7568705629914568, + 0.7052581468908609, + 0.8265948292968273, + 0.8314431612258691, + 0.8367812897894294, + 0.7830341552846619, + 0.9260397306909466, + 0.8976492774311802, + 0.6709612120262689, + 0.6409396659579042, + 0.926759644425545, + 0.6910497300932421, + 0.8926655526767395, + 0.7268402166505129, + 0.6674110053707871, + 0.6748167363333102 + ], + "mean_band_energy": [ + 2.9020119969008866, + 3.373746097486265, + 4.16020192466315, + 5.953360286002187, + 3.074453386789567, + 3.224726005218658, + 4.136820051277348, + 5.735619276464023, + 13.591993971929387, + 15.208044459971266, + 14.638813866529222, + 16.770395600731533, + 16.659114534179636, + 14.825602460502457, + 14.046907553020164, + 15.922847167971348, + 13.39892540712954, + 15.59149694165388, + 15.276331656072758, + 12.757932497542681, + 15.29199534825657, + 15.749383287445648, + 15.182964782255087, + 15.716489705049614, + 14.431057701644784, + 15.342294047798955, + 15.341634640021233, + 13.98280810884469, + 15.029542227530182, + 14.419719804620604, + 14.93802321175704, + 15.560574405262104 + ], + "dft_magnitude": [ + 392.2358324125225, + 78.56264532886391, + 58.31926598782952, + 25.535980911819735, + 2.398275593321554, + 18.839404225028613, + 15.96710167480629, + 18.84589002012652, + 5.738932884188103, + 5.966777845413461, + 16.02907973117695, + 8.64369059586531, + 9.47998433467877, + 9.473118576883627, + 10.309481446678843, + 4.932825731548186, + 8.034247892609358 + ], + "mean_band_dot": [ + 0.0002034762929952194, + 0.0001356051943730563, + 5.279378910927335e-05, + 0.00013603981915366603, + 0.00019318902585041542, + 5.302660156303318e-05, + 7.473659934476018e-05, + 0.0001824172145461489, + 0.0004125516275053087, + 0.000505044506098784, + 0.0005258961600702605, + 0.00045955685527587775, + 0.00044956758256375906, + 0.0004403149750942248, + 0.00048252649139612913, + 0.00043381331670389045, + 0.0005297941734170308, + 0.00048353790225519333, + 0.0004410647707118187, + 0.0004443074367372901, + 0.00046739395270378736, + 0.00038725470335521095, + 0.0004951930718561925, + 0.0004946959415974561, + 0.00045428786370393937, + 0.0003896538779599723, + 0.0003970999650846352, + 0.00040406999141850974, + 0.0005313085262059758, + 0.0004882945738700073, + 0.0005004754975743708, + 0.0005376389906359691 + ] + }, + { + "layer": 22, + "mean_snr": 0.782921324832916, + "per_head_snr": [ + 0.7238046680669791, + 0.7440308130293296, + 0.8365803179463115, + 1.0097427226810458, + 0.724985744043335, + 0.7344077848825846, + 0.9311128023008992, + 0.7417496745759581, + 0.6289466012322023, + 0.6707616937663414, + 0.7911893397914301, + 0.7464037513240005, + 0.7985855184301657, + 0.7630085538224931, + 0.8398216598260023, + 0.8416095516075769 + ], + "mean_band_energy": [ + 4.791786130184399, + 5.143650645779417, + 6.590424960351152, + 9.24559669526784, + 4.775997356294934, + 5.517734730505592, + 5.8749426914552885, + 8.990006481131816, + 17.671932495347782, + 17.28488807550206, + 17.44229216921096, + 15.80318646864233, + 16.853853584239857, + 15.297998300911615, + 15.6836001263232, + 18.892619770864854, + 17.904698863344187, + 17.197512842330127, + 17.49794544458814, + 15.628001542278703, + 17.76233146797471, + 16.463693884305478, + 18.170107037981133, + 16.75675063397599, + 18.578431597853477, + 17.196502185330882, + 16.31206881094659, + 17.422816232499862, + 17.6377601441082, + 17.354291677383973, + 18.81731887407033, + 16.77098263165307 + ], + "dft_magnitude": [ + 463.3317245526379, + 77.39850555764875, + 54.08480548094067, + 28.403579523761138, + 5.995490931053999, + 25.953074050925604, + 22.998359286087133, + 16.502888717520154, + 8.064214830602719, + 9.548330533064943, + 15.517284079128531, + 2.7627857882432894, + 1.973950039287463, + 8.028063059449327, + 11.829606760846117, + 9.997357843584911, + 1.3992589559107387 + ], + "mean_band_dot": [ + 0.00016498379056884005, + 9.527514066576259e-05, + 0.00031905182254376996, + 0.0003068383448407985, + 8.110095245683624e-05, + 0.00017813652686982095, + 0.0002031160709066171, + 0.00022225050497581833, + 0.0006779099157938617, + 0.0007356444521064986, + 0.0007038018075036234, + 0.0006825567515988951, + 0.0006621127631660784, + 0.0005576101498263597, + 0.0007390719592876849, + 0.0007391086601273855, + 0.0007544298332504695, + 0.000672410621518793, + 0.0006661217830696842, + 0.0005944160202489002, + 0.0007299687349586748, + 0.0006874204991618171, + 0.0007334057668231253, + 0.0007590864106532536, + 0.0007004395065450808, + 0.000742121447729005, + 0.0007033356732790708, + 0.0007411315436911536, + 0.0007473155928892083, + 0.0007846155208426353, + 0.0007215510395326419, + 0.0006368579122408846 + ] + }, + { + "layer": 23, + "mean_snr": 0.7875591395414828, + "per_head_snr": [ + 0.7407848751005296, + 0.9136514987318256, + 0.7763769891530174, + 0.8176878548649065, + 0.6923328058708256, + 0.7452988821159091, + 0.8470101539753158, + 0.7206203257483587, + 1.0280380671488036, + 0.7524224320908453, + 0.6975628299399866, + 0.7221743849427087, + 0.7827488301639672, + 0.8916388113863759, + 0.7150786529639329, + 0.757518838466416 + ], + "mean_band_energy": [ + 5.2370412128218575, + 5.3257449729394315, + 6.328708138956751, + 8.455624377180433, + 5.316729734816557, + 5.618958145104883, + 6.91213245485109, + 8.311443629746465, + 17.752778756728087, + 18.093844866459065, + 17.102528509728547, + 16.054687237588947, + 17.232581973009545, + 16.630353452764698, + 18.293879532612934, + 18.14444747269067, + 16.771712575506857, + 17.35874424154933, + 19.62366090443438, + 17.897218638210383, + 18.716377308826722, + 18.617746614378433, + 17.87578203900115, + 18.691755332551118, + 17.22543889477003, + 17.33912330531467, + 17.537083635437014, + 17.305115104627703, + 17.482785905420663, + 17.075318527908806, + 18.676613558907516, + 18.5405028938482 + ], + "dft_magnitude": [ + 477.546463948693, + 84.09317693001084, + 50.77940376843491, + 32.28289556320634, + 4.028414228264251, + 16.30685904738983, + 26.43449762021449, + 16.678282310151534, + 9.881658011604014, + 9.380337774420108, + 15.836754762876632, + 5.949455151250441, + 4.936531073094448, + 10.122296131335894, + 8.76050693218412, + 10.160708664189135, + 1.3747936770335514 + ], + "mean_band_dot": [ + 6.116254940025101e-05, + 0.00017486790807197394, + 0.00029690675455640303, + 0.0002553275007812772, + 0.00019488845418891287, + 0.0002908747835590475, + 0.00037931293957171874, + 0.00026313013631806825, + 0.0004882093389824149, + 0.0005975560925435275, + 0.0005318740313668968, + 0.000518688957527047, + 0.0006687230525130872, + 0.0006013990314386319, + 0.0006555645923072007, + 0.0004929141841785167, + 0.000543897398529225, + 0.000488200775180303, + 0.0006476419912360143, + 0.0006296960718827904, + 0.0005730790035158861, + 0.00043533517919058795, + 0.0006200220796017675, + 0.0006411352960640215, + 0.0006152389323688112, + 0.0006139310828530142, + 0.0004978533188477741, + 0.0005595039940544666, + 0.0005665017029059527, + 0.0005542760563912452, + 0.0006262977931328351, + 0.0006436569201468956 + ] + } + ], + "elapsed_s": 0.40142321586608887 +} \ No newline at end of file diff --git a/data/exp_wqk_spectral/pythia-70m_wqk_spectral.json b/data/exp_wqk_spectral/pythia-70m_wqk_spectral.json new file mode 100644 index 0000000000000000000000000000000000000000..24062297e49b7662383119e0be0537dede7cdab0 --- /dev/null +++ b/data/exp_wqk_spectral/pythia-70m_wqk_spectral.json @@ -0,0 +1,621 @@ +{ + "model": "EleutherAI/pythia-70m", + "short_name": "pythia-70m", + "theta": 10000, + "T_train": 2048, + "d_head": 64, + "n_pairs": 32, + "k_dead": 23, + "global_mean_snr": 0.8728831521760515, + "global_min_snr": 0.7859839851544951, + "global_max_snr": 0.9644368280316371, + "layers": [ + { + "layer": 0, + "mean_snr": 0.844582022516127, + "per_head_snr": [ + 0.8305933580986826, + 0.9592422506633252, + 0.7885759082483262, + 0.7519907149036857, + 0.867178151082206, + 0.873491127103617, + 0.8604318988543304, + 0.8251527711748431 + ], + "mean_band_energy": [ + 0.7216495517290231, + 0.7788312365613499, + 0.7875515910367785, + 1.434534914230973, + 0.8363050732025776, + 0.868145060088032, + 0.950716354102674, + 1.4471906559466792, + 1.871799024874342, + 1.8667119071863016, + 1.8586622728442461, + 1.752083736477605, + 1.8599083282215312, + 1.818449648281152, + 1.8047409596146493, + 1.9013249194008295, + 1.8727404077350953, + 1.848480669955888, + 1.8601044568242648, + 1.8772003938885993, + 1.8498166812633547, + 1.775661581669866, + 1.8416582621946982, + 1.8581192556926283, + 1.8415773811873413, + 1.8477950909609966, + 1.87234710838568, + 1.8271381035888354, + 1.748040098120156, + 1.8888700950422206, + 1.9079423043190284, + 1.8446552759217076 + ], + "dft_magnitude": [ + 52.12075240054911, + 6.261159560988024, + 4.424820246172176, + 2.3557781538879667, + 0.27423297126916507, + 1.5980776672175867, + 2.4286807260340253, + 1.9845157339048622, + 1.2807090020397363, + 0.5610760214930772, + 0.40776910919410503, + 0.12499989182308917, + 0.09331733790668413, + 0.48801609588580636, + 1.0777015896011544, + 1.2353847464236924, + 1.1496326892382278 + ], + "mean_band_dot": [ + -0.00013309496620195205, + 3.069219974349835e-05, + 8.455746353774884e-06, + 5.334954425961769e-05, + -0.00011078188163082814, + -8.871601812643348e-05, + 4.872067290762061e-05, + 7.978801704666694e-05, + 0.00017427288275939645, + 9.832217074290384e-05, + 0.00016990491076285252, + 0.00018049351456284057, + 0.00017932664377440233, + 0.00011653332148853224, + 0.00017711505824991036, + 0.00011074423309764825, + 0.00014079950142331654, + 0.0001883548511614208, + 0.00020895687703159638, + 0.0001575931901243166, + 0.00018918959676739178, + 0.000199970373614633, + 0.00013663794015883468, + 0.00011092581917182542, + 0.00015523001093242783, + 0.0001124959162552841, + 0.00018463230389897944, + 0.0002819869440600087, + 0.00018373365401203046, + 0.00013475658215611475, + 0.00010554904019954847, + 0.00022076237291912548 + ] + }, + { + "layer": 1, + "mean_snr": 0.9102494921785727, + "per_head_snr": [ + 0.9085675942244394, + 0.9192446739430464, + 0.8372796939901613, + 1.0576991797160022, + 0.9144384632691951, + 0.9199590181022559, + 0.8591503366054215, + 0.8656569775780588 + ], + "mean_band_energy": [ + 1.5284769957405504, + 1.486628152848116, + 1.6703863897753468, + 2.1726826093832265, + 1.3923949050757827, + 1.5632273577756335, + 1.8112354220299505, + 2.2437877103980863, + 2.59524132413625, + 2.5544664422792147, + 2.5881548810948063, + 2.528223716237153, + 2.5962057384111947, + 2.601447257138128, + 2.510038824666519, + 2.555943143801662, + 2.593465047202456, + 2.5961320259304586, + 2.5890535557534875, + 2.569330713831145, + 2.6094290326928973, + 2.590030624556811, + 2.5486324543265315, + 2.5697708476381305, + 2.5871059689144538, + 2.5620227574043932, + 2.6019669401334298, + 2.5407804210536806, + 2.584891058815206, + 2.575681505963957, + 2.5861373722198837, + 2.537772377373736 + ], + "dft_magnitude": [ + 75.64074357460228, + 6.178946948123061, + 4.419387082980736, + 2.432852374954648, + 0.1932157766556644, + 1.2201771875328853, + 1.9815725361188068, + 2.051502490461307, + 1.260141546448223, + 1.1050258643147584, + 0.6975141448803707, + 0.3975026836422874, + 0.21245486623113888, + 0.6196377994708513, + 0.979277237866785, + 0.9572548720050369, + 0.8551117526247864 + ], + "mean_band_dot": [ + -1.954571757778467e-05, + 2.6530635523158708e-05, + 2.2198680653673364e-06, + -8.257906347353128e-05, + -2.6666597705116146e-05, + 6.44031943011214e-05, + 9.692641560832271e-05, + -1.3885026419302449e-05, + 1.3764316122433229e-05, + -1.1082684068242088e-05, + 2.835065106410184e-05, + -2.2031000298738945e-05, + 8.087257265287917e-05, + 3.068258138227975e-06, + 9.144740488409298e-06, + -5.469410916703055e-05, + -3.517048980938853e-05, + -1.6993636563711334e-05, + -2.5539069838487194e-05, + 2.069824404316023e-05, + 3.5150322673871415e-05, + 2.9592124519695062e-06, + -2.604354403956677e-05, + 5.786576639366103e-05, + -1.0153416951652616e-05, + 2.862322298824438e-05, + 4.352402606855321e-05, + 7.672508036193904e-05, + -1.103917838918278e-05, + 1.2754021383898362e-05, + 2.698462594707962e-05, + 5.7997161775347195e-06 + ] + }, + { + "layer": 2, + "mean_snr": 0.9644368280316371, + "per_head_snr": [ + 0.9762505475061739, + 0.9435257928077422, + 1.020138603320756, + 0.965942380595845, + 1.0260740285307965, + 0.9071268557155155, + 0.8825746886952027, + 0.993861727081065 + ], + "mean_band_energy": [ + 0.9374102561787567, + 1.302321605800955, + 1.2924807327952257, + 1.4775406236383786, + 0.9529583531550534, + 1.3561612157169085, + 1.292254199514407, + 1.4138055646476504, + 1.5275489686325625, + 1.529938313573827, + 1.5010531617229934, + 1.4734719873928808, + 1.4537343234705276, + 1.474108819736343, + 1.5044181471163443, + 1.4448465027162674, + 1.4953034396064644, + 1.513688268809286, + 1.4456231563247206, + 1.426497818591383, + 1.4818784916709897, + 1.4849708239510218, + 1.5244598269711944, + 1.500534321280531, + 1.485680360000233, + 1.4972082573428436, + 1.5011331419729483, + 1.446789384077638, + 1.4959858025390238, + 1.4350284470309917, + 1.4245061998839126, + 1.4289633001220339 + ], + "dft_magnitude": [ + 45.522303815984294, + 1.6943910647607123, + 1.290340698546117, + 0.42414742427895197, + 0.13297083567084192, + 0.7224123986928821, + 0.6294921371199825, + 0.6887154295371416, + 0.6557045941805362, + 0.6876185890843356, + 0.49824165240418794, + 0.5464586690527408, + 0.09890481467311596, + 0.44469187162309526, + 0.7954171337022063, + 1.004880142235326, + 0.8894466928735838 + ], + "mean_band_dot": [ + -5.997216840114561e-05, + 9.115548527915962e-05, + 7.488734649996331e-05, + 5.872564315723139e-05, + -6.181888602441177e-05, + 9.645043974160217e-06, + 7.119020946788623e-05, + 0.0001095667912522913, + 0.0002712942150537856, + 0.00026304636730856146, + 0.00024363554030060186, + 0.00022460246464106604, + 0.00019342681844136678, + 0.00022391254651665804, + 0.0002766432576208899, + 0.00018331281671635224, + 0.00022746691388420004, + 0.00028708413958611345, + 0.00019376104728507926, + 0.0001655549602901374, + 0.00026003353241321747, + 0.00025344968855733896, + 0.0002726191623878549, + 0.00027443434896667895, + 0.00024086534949674387, + 0.00023384492487821262, + 0.00024928236734922393, + 0.00023692188688073657, + 0.00025976824986173597, + 0.00018988731306990303, + 0.00016593219834248885, + 0.00014699244263738365 + ] + }, + { + "layer": 3, + "mean_snr": 0.8883676577561075, + "per_head_snr": [ + 0.905928702884333, + 0.6935978110277735, + 1.024358661086641, + 0.9848402496084461, + 1.023088611877425, + 0.8030230612347716, + 0.7310209019055095, + 0.9410832624239599 + ], + "mean_band_energy": [ + 1.4176264584879266, + 2.2903519615819548, + 2.5670976447842024, + 3.9327150672377016, + 1.2597150165657984, + 2.310469243950232, + 2.7121367810700168, + 3.8095966784682407, + 5.637457052257167, + 5.811410305276958, + 7.00197960018609, + 5.01951910313308, + 6.593776846950506, + 5.3009111850786415, + 5.050593931248942, + 5.376486023082677, + 5.80572967293857, + 6.095647776396479, + 6.297003869475544, + 6.519341330824286, + 8.870913401571332, + 4.287745843889397, + 5.752452077736127, + 6.2508430642744255, + 4.912977407755182, + 5.549379939251882, + 5.169064714940978, + 6.183032093548169, + 5.192282533603175, + 6.442462898053094, + 5.647835275018034, + 6.54513994177033 + ], + "dft_magnitude": [ + 161.61369474040714, + 24.779292393206433, + 13.320834697590696, + 14.988865472632536, + 3.9309222562416513, + 4.028748751551764, + 8.353766328919287, + 4.339668101962337, + 5.571473102652847, + 9.179242819628014, + 1.9186906170638718, + 4.711812179855826, + 5.481642615106611, + 5.980358228261637, + 5.042719842373983, + 10.837881129889286, + 1.8364101712279535 + ], + "mean_band_dot": [ + -0.00029629239134010277, + -0.00023290766330319457, + -0.00020351371199467394, + 0.00041104260526481085, + -0.00025646211270213826, + -6.970497679503751e-05, + -0.00019987846371805063, + 0.0004732330507977167, + 0.0005028589257562999, + 0.0003430877568462165, + 0.0004593763478624169, + 0.0006557246260854299, + 0.0004235931835410156, + 0.0005590620012299041, + 0.00046927760195103474, + 0.0005601864040727378, + 0.00046120499973767437, + 0.000567441975363181, + 0.0006179927477205638, + 0.00046340791232069023, + 0.0004181291806162335, + 0.0004029861229355447, + 0.00040939925384009257, + 0.00038623399541393155, + 0.0004413876513353898, + 0.0006985014083511487, + 0.0005289764376357198, + 0.0005241385206318228, + 0.00042159697477472946, + 0.0003523583645801409, + 0.0004954368844209966, + 0.00041477270315226633 + ] + }, + { + "layer": 4, + "mean_snr": 0.8436789274193698, + "per_head_snr": [ + 0.7699596250082249, + 0.8709207368787065, + 0.6437189375510503, + 0.619608486553649, + 0.9705677855393122, + 1.18162103845269, + 0.9924982206446481, + 0.7005365887266773 + ], + "mean_band_energy": [ + 6.616067408919349, + 9.2332424639676, + 11.496448589224602, + 18.167734333602183, + 5.99537101758802, + 10.482523404249406, + 11.9859882471956, + 18.66996603797326, + 28.02584890469413, + 31.36077578597167, + 28.66642336498103, + 35.43821041713284, + 33.31016299353411, + 51.63041586956072, + 41.13461152448994, + 31.619746532359642, + 38.3914550112234, + 49.683819201692486, + 35.17855415715037, + 33.229398075055215, + 26.954449132094354, + 56.882300099717526, + 44.0456008922702, + 27.661760392189677, + 35.718809200470986, + 31.038330639294372, + 39.23169159122408, + 26.703663106105125, + 49.146156307687164, + 31.58403711683887, + 45.69328134132945, + 35.71450696261371 + ], + "dft_magnitude": [ + 980.691350122401, + 204.83307710756986, + 120.38795732375857, + 52.194654468859525, + 56.13024133451598, + 45.1934763809744, + 23.519570461193382, + 90.12309488393434, + 55.717275531429365, + 52.70950303391818, + 37.27380772298016, + 38.507941966138496, + 15.839016024092002, + 52.117128184692035, + 60.3905860479565, + 52.462989804908815, + 17.509510754247515 + ], + "mean_band_dot": [ + -0.0008636987572572252, + -0.0009205255937558832, + -0.0013973088389320765, + 0.00017211735394084826, + -0.0010227745879092254, + -0.0009650753745518159, + -0.0010157252963836072, + 0.00019854885522363475, + 0.0015016677334642736, + 0.001242763435584493, + 0.0012837423146265792, + 0.0016866312325873878, + 0.002170407293306198, + 0.0014163157393340953, + 0.0010914971517195227, + 0.0009470036193306441, + 0.0009022724643727997, + 0.001625149125175085, + 0.0011589213463594206, + 0.0012902590779049206, + 0.001388656555718626, + 0.0012417729340086225, + 0.0010690633607737254, + 0.0011092982422269415, + 0.0011461066224001115, + 0.0010383290555182612, + 0.0015956421975715784, + 0.0014619334342569346, + 0.0016048822326411027, + 0.0013783265894744545, + 0.0010581319038465153, + 0.0014643270878877956 + ] + }, + { + "layer": 5, + "mean_snr": 0.7859839851544951, + "per_head_snr": [ + 0.7259533056451443, + 0.8520949195172665, + 0.7324619170767037, + 0.8812716698245258, + 0.8506935579970591, + 0.8689241728185525, + 0.7770946851721352, + 0.5993776531845737 + ], + "mean_band_energy": [ + 16.657220539700802, + 19.960963941636237, + 22.84179942564417, + 25.703561774654005, + 15.078156688781604, + 19.167820596607207, + 22.82881754506451, + 29.990542316457322, + 36.03709296882943, + 45.069015466464975, + 50.5045452047417, + 44.88419231637948, + 43.44959508246578, + 36.10494670892345, + 38.90563158068201, + 37.78111559060201, + 39.73207303346115, + 36.90137670590087, + 39.49202049881899, + 37.96322256496734, + 35.160804232424454, + 45.6160864661731, + 42.95226108244205, + 49.698480493498565, + 45.875849916401734, + 44.11783843335826, + 37.51015807618214, + 39.893142451888906, + 34.02475752820749, + 38.89863164630867, + 53.490789867899764, + 46.17388074518084 + ], + "dft_magnitude": [ + 1172.4663914907492, + 128.72985208667407, + 120.43104525495147, + 76.69131703002043, + 29.394114375703694, + 65.7859395609945, + 41.44215150278272, + 38.40286618212025, + 49.96278016385896, + 33.32044739990712, + 28.221077040045056, + 20.049878443617065, + 4.8284883764616815, + 11.416938602480595, + 21.89956557214787, + 24.84199497258902, + 23.383244947253502 + ], + "mean_band_dot": [ + -0.0004445515078259632, + 0.0014525553051498719, + -0.0011358756482877652, + -0.0013281297506182455, + 0.0011471294274087995, + 0.00047052627633092925, + -0.0009797487291507423, + -0.0016303694810630986, + 0.0005248263405519538, + -0.0003929642989533022, + 0.00018124964117305353, + 0.00047952223030733876, + -0.00036599146187654696, + -6.937080979696475e-05, + -0.0011017534034181153, + 5.8562618505675346e-06, + 0.00011732837083400227, + 0.0002965669364130008, + -0.0007683121075388044, + 0.0007900803648226429, + 0.001164732180768624, + -0.0005064209790361929, + -0.0007214589641080238, + 0.00010191334513365291, + -0.00030551067720807623, + -0.00030649696782347746, + -0.0001689532982709352, + 0.0006336825863400009, + -0.00021970345187583007, + -0.0005278909593471326, + -0.00014291565912571969, + -0.00035880969880963676 + ] + } + ], + "elapsed_s": 0.04682016372680664 +} \ No newline at end of file diff --git a/data/exp_yarn/pythia-70m_yarn_scan.json b/data/exp_yarn/pythia-70m_yarn_scan.json new file mode 100644 index 0000000000000000000000000000000000000000..4a8fd792548db408fc0de4443ddddfcbd156270e --- /dev/null +++ b/data/exp_yarn/pythia-70m_yarn_scan.json @@ -0,0 +1,88 @@ +{ + "model": "pythia-70m", + "d_head": 64, + "theta_opt": 25330, + "n_prompts": 30, + "results": { + "orig": { + "theta": 10000, + "f_active": 0.4688, + "R_curve": [ + -0.6016, + -0.2562, + -1.0246, + 1.2354, + 0.9883, + 1.0 + ], + "L_crit": 3, + "max_R": 1.2354, + "mu_baseline": -14.133, + "mu_d10": -13.5259 + }, + "2x": { + "theta": 20000, + "f_active": 0.5, + "R_curve": [ + -1.2888, + -0.6825, + -0.4616, + 0.8958, + 1.0183, + 1.0 + ], + "L_crit": 4, + "max_R": 1.0183, + "mu_baseline": -14.7723, + "mu_d10": -14.1086 + }, + "5x": { + "theta": 50000, + "f_active": 0.5625, + "R_curve": [ + 3.1073, + 3.6146, + 2.2174, + 0.9883, + 0.9279, + 1.0 + ], + "L_crit": 0, + "max_R": 3.6146, + "mu_baseline": -14.2183, + "mu_d10": -14.6064 + }, + "10x": { + "theta": 100000, + "f_active": 0.5625, + "R_curve": [ + 14.4179, + 14.652, + 7.5319, + 0.4504, + 2.0614, + 1.0 + ], + "L_crit": 0, + "max_R": 14.652, + "mu_baseline": -14.4646, + "mu_d10": -14.5444 + }, + "opt": { + "theta": 25330, + "f_active": 0.5, + "R_curve": [ + -0.9232, + -0.0786, + -0.7304, + 0.9235, + 0.8343, + 1.0 + ], + "L_crit": 3, + "max_R": 1.0, + "mu_baseline": -14.8087, + "mu_d10": -14.261 + } + } +} \ No newline at end of file diff --git a/data/falcon_alibi/falcon_alibi_fit.json b/data/falcon_alibi/falcon_alibi_fit.json new file mode 100644 index 0000000000000000000000000000000000000000..fae836c97a0fc7c0fd616dd7f7371cb3296167e8 --- /dev/null +++ b/data/falcon_alibi/falcon_alibi_fit.json @@ -0,0 +1,30 @@ +{ + "mongo": { + "power_law": { + "gamma": 0.924684527882036, + "r2": 0.9897280156969657 + }, + "model_c": { + "C0": 0.038565128711944834, + "gamma": 0.7439435362461131, + "lambda": 0.0010386588588366962, + "decay_scale_tokens": 962.7800230000499, + "r2": 0.9989847819090889 + }, + "interpretation": "power-law dominates" + }, + "random": { + "power_law": { + "gamma": 0.9140832027359694, + "r2": 0.9678067134098391 + }, + "model_c": { + "C0": 0.009807975311113562, + "gamma": 0.29756654703754865, + "lambda": 0.005102979221976389, + "decay_scale_tokens": 195.963956837884, + "r2": 0.9942497317969279 + }, + "interpretation": "ALiBi exp dominates" + } +} \ No newline at end of file diff --git a/data/gamma_dial_gpu/pythia-70m_gamma_dial.json b/data/gamma_dial_gpu/pythia-70m_gamma_dial.json new file mode 100644 index 0000000000000000000000000000000000000000..7456b538e2ef6bb5929037c134d26f34c2ef3f3d --- /dev/null +++ b/data/gamma_dial_gpu/pythia-70m_gamma_dial.json @@ -0,0 +1,92 @@ +{ + "model": "pythia-70m", + "k_dead": 23, + "distances": [ + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "results": { + "1.0": { + "alpha": 1.0, + "k_dead": 23, + "means": [ + 0.009425997069726388, + 0.007616706314341475, + 0.003960159035632387, + 0.0018816962791849314, + 0.0014853083784206924, + 0.0006055987625450143, + 0.0003808813685085018, + 0.00015858603301590317 + ], + "gamma_real": 0.7692273954314717, + "r2": 0.9860517153515315, + "gamma_sage_meanfield": 0.33, + "err_vs_meanfield_pct": 133.0992107368096, + "ppl_wikitext": NaN + }, + "2.0": { + "alpha": 2.0, + "k_dead": 23, + "means": [ + 0.00890763213156788, + 0.006072100852854874, + 0.0031495152239991364, + 0.0017093524947366728, + 0.001437205056663894, + 0.0005462497385639863, + 0.00045400904272921617, + 0.00015552891511949496 + ], + "gamma_real": 0.7273015822544925, + "r2": 0.9815381835461346, + "gamma_sage_meanfield": 0.48, + "err_vs_meanfield_pct": 51.52116296968594, + "ppl_wikitext": NaN + }, + "3.0": { + "alpha": 3.0, + "k_dead": 23, + "means": [ + 0.009233167045204407, + 0.004930038112424881, + 0.0035480961872745333, + 0.002592464875204937, + 0.001741171412087207, + 0.000496885728552936, + 0.00040662634407486413, + 0.0002976683259442052 + ], + "gamma_real": 0.6676490038337431, + "r2": 0.9715798143918007, + "gamma_sage_meanfield": 0.69, + "err_vs_meanfield_pct": 3.2392748067038837, + "ppl_wikitext": NaN + }, + "5.0": { + "alpha": 5.0, + "k_dead": 23, + "means": [ + 0.010232932424529889, + 0.005207723380084042, + 0.004237773541388871, + 0.004040550790293906, + 0.0009580180254268576, + 0.0012850861333382796, + 0.0005448017653475767, + 0.00015817918144421132 + ], + "gamma_real": 0.7022015830608412, + "r2": 0.9060310930476675, + "gamma_sage_meanfield": 1.03, + "err_vs_meanfield_pct": 31.8250890232193, + "ppl_wikitext": NaN + } + } +} \ No newline at end of file diff --git a/data/kuramoto/EleutherAI--pythia-160m_kuramoto.json b/data/kuramoto/EleutherAI--pythia-160m_kuramoto.json new file mode 100644 index 0000000000000000000000000000000000000000..164dd0997bd3397753447f03c2aa777927f5fd4a --- /dev/null +++ b/data/kuramoto/EleutherAI--pythia-160m_kuramoto.json @@ -0,0 +1,41 @@ +{ + "model": "EleutherAI/pythia-160m", + "N": 12, + "theta": 10000, + "d_head": 64, + "K_c": 0.73255, + "Omega": 0.12493, + "K_layers": [ + 0.73318, + 0.74533, + 0.74341, + 0.74635, + 0.65938, + 1.04304, + 1.44243, + 3.63134, + 4.44416, + 8.37846, + 11.36864, + 9.21615 + ], + "cumK": [ + 0.73318, + 0.73926, + 0.74064, + 0.74207, + 0.72553, + 0.77845, + 0.8733, + 1.21806, + 1.57651, + 2.25671, + 3.08507, + 3.59599 + ], + "L_crit_kura_peak": 10, + "L_crit_kura_jump": 7, + "alpha_peak": 0.8333, + "L_crit_h3": 3.5, + "alpha_h3": 0.2917 +} \ No newline at end of file diff --git a/data/kuramoto/EleutherAI--pythia-1b_kuramoto.json b/data/kuramoto/EleutherAI--pythia-1b_kuramoto.json new file mode 100644 index 0000000000000000000000000000000000000000..4298275499ba4caa856f256e4e9c846fae4e3a52 --- /dev/null +++ b/data/kuramoto/EleutherAI--pythia-1b_kuramoto.json @@ -0,0 +1,49 @@ +{ + "model": "EleutherAI/pythia-1b", + "N": 16, + "theta": 10000, + "d_head": 256, + "K_c": 0.65973, + "Omega": 0.11252, + "K_layers": [ + 1.07493, + 1.22168, + 1.17802, + 1.34078, + 1.1564, + 1.06412, + 1.05706, + 1.1284, + 1.04504, + 1.08163, + 1.05499, + 1.01059, + 1.14413, + 1.08995, + 1.19469, + 1.20779 + ], + "cumK": [ + 1.07493, + 1.14831, + 1.15821, + 1.20385, + 1.19436, + 1.17265, + 1.15614, + 1.15267, + 1.14071, + 1.13481, + 1.12755, + 1.1178, + 1.11983, + 1.11769, + 1.12283, + 1.12814 + ], + "L_crit_kura_peak": 3, + "L_crit_kura_jump": null, + "alpha_peak": 0.1875, + "L_crit_h3": 15.0, + "alpha_h3": 0.9375 +} \ No newline at end of file diff --git a/data/kuramoto/EleutherAI--pythia-70m_kuramoto.json b/data/kuramoto/EleutherAI--pythia-70m_kuramoto.json new file mode 100644 index 0000000000000000000000000000000000000000..f19b11c684e528304d0cd1ca44561cc0ee23c508 --- /dev/null +++ b/data/kuramoto/EleutherAI--pythia-70m_kuramoto.json @@ -0,0 +1,29 @@ +{ + "model": "EleutherAI/pythia-70m", + "N": 6, + "theta": 10000, + "d_head": 64, + "K_c": 0.73255, + "Omega": 0.12493, + "K_layers": [ + 0.0669, + 0.06715, + 0.06386, + 0.17828, + 0.29539, + 0.27403 + ], + "cumK": [ + 0.0669, + 0.06703, + 0.06597, + 0.09405, + 0.13432, + 0.1576 + ], + "L_crit_kura_peak": 4, + "L_crit_kura_jump": 3, + "alpha_peak": 0.6667, + "L_crit_h3": 4.0, + "alpha_h3": 0.6667 +} \ No newline at end of file diff --git a/data/kuramoto/gpt2-large_kuramoto.json b/data/kuramoto/gpt2-large_kuramoto.json new file mode 100644 index 0000000000000000000000000000000000000000..e91830f8199ca4d467756432e5941a81a6f69175 --- /dev/null +++ b/data/kuramoto/gpt2-large_kuramoto.json @@ -0,0 +1,89 @@ +{ + "model": "gpt2-large", + "N": 36, + "theta": 10000.0, + "d_head": 64, + "K_c": 0.73255, + "Omega": 0.12493, + "K_layers": [ + 0.81441, + 0.75387, + 0.62833, + 0.60773, + 0.61758, + 0.62828, + 0.63754, + 0.63669, + 0.6088, + 0.64065, + 0.60044, + 0.56273, + 0.54459, + 0.54527, + 0.57492, + 0.5152, + 0.45617, + 0.45015, + 0.44311, + 0.43883, + 0.43698, + 0.44074, + 0.42647, + 0.45656, + 0.44064, + 0.42717, + 0.4223, + 0.43501, + 0.42585, + 0.42134, + 0.39643, + 0.42332, + 0.40557, + 0.44298, + 0.46508, + 0.53274 + ], + "cumK": [ + 0.81441, + 0.78414, + 0.73221, + 0.70109, + 0.68439, + 0.67503, + 0.66968, + 0.66555, + 0.65925, + 0.65739, + 0.65221, + 0.64476, + 0.63705, + 0.63049, + 0.62679, + 0.61982, + 0.61019, + 0.6013, + 0.59297, + 0.58527, + 0.5782, + 0.57196, + 0.56563, + 0.56109, + 0.55627, + 0.5513, + 0.54652, + 0.54254, + 0.53852, + 0.53461, + 0.53015, + 0.52682, + 0.52314, + 0.52078, + 0.51919, + 0.51957 + ], + "L_crit_kura_peak": 0, + "L_crit_kura_jump": null, + "alpha_peak": 0.0, + "L_crit_h3": 35.0, + "alpha_h3": 0.9722 +} \ No newline at end of file diff --git a/data/kuramoto/gpt2-medium_kuramoto.json b/data/kuramoto/gpt2-medium_kuramoto.json new file mode 100644 index 0000000000000000000000000000000000000000..d4c269cbe8c105ac76fe50debce0bb82ddf4af84 --- /dev/null +++ b/data/kuramoto/gpt2-medium_kuramoto.json @@ -0,0 +1,65 @@ +{ + "model": "gpt2-medium", + "N": 24, + "theta": 10000.0, + "d_head": 64, + "K_c": 0.73255, + "Omega": 0.12493, + "K_layers": [ + 79.32437, + 15.34827, + 18.48681, + 20.93346, + 21.07316, + 19.14979, + 15.20595, + 13.88462, + 13.71983, + 12.67216, + 13.36523, + 12.39064, + 12.63975, + 13.46177, + 12.51924, + 11.95748, + 12.09888, + 11.69769, + 11.62254, + 10.59987, + 10.37043, + 10.22616, + 9.51778, + 10.41475 + ], + "cumK": [ + 79.32437, + 47.33632, + 37.71981, + 33.52323, + 31.03321, + 29.05264, + 27.07454, + 25.4258, + 24.12514, + 22.97984, + 22.10579, + 21.29619, + 20.63031, + 20.11827, + 19.61167, + 19.13328, + 18.71949, + 18.32939, + 17.9764, + 17.60758, + 17.26295, + 16.94309, + 16.62026, + 16.36169 + ], + "L_crit_kura_peak": 0, + "L_crit_kura_jump": null, + "alpha_peak": 0.0, + "L_crit_h3": 23.0, + "alpha_h3": 0.9583 +} \ No newline at end of file diff --git a/data/learned_scalars/pythia-70m_learned_scalars.json b/data/learned_scalars/pythia-70m_learned_scalars.json new file mode 100644 index 0000000000000000000000000000000000000000..4ec53b888254fddad3f28d98c9b172a5e5f0b353 --- /dev/null +++ b/data/learned_scalars/pythia-70m_learned_scalars.json @@ -0,0 +1,1249 @@ +{ + "model": "pythia-70m", + "k_dead": 23, + "n_pairs": 32, + "gamma_baseline": 0.7493589625280966, + "r2_baseline": 0.9868773001810779, + "targets": { + "0.5": { + "gamma_after": 0.6860529756864402, + "r2_after": 0.9895254851662268, + "gamma_delta": -0.06330598684165645, + "alpha_per_layer": [ + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 0.5871334075927734, + 2.5806350708007812, + 0.629039466381073, + 0.5857845544815063, + 0.5989060401916504, + 2.5404629707336426, + 0.6509703993797302, + 0.844393789768219, + 1.1123955249786377, + 1.0720241069793701, + 0.9867271780967712, + 1.0758463144302368, + 1.0822213888168335, + 1.12770414352417, + 1.0284141302108765, + 1.1603379249572754, + 1.0743451118469238, + 1.0915288925170898, + 1.0022770166397095, + 1.0705503225326538, + 0.986470103263855, + 1.1722437143325806, + 1.1463810205459595, + 0.9713058471679688, + 1.081142783164978, + 1.0614659786224365, + 1.0428673028945923, + 1.02228844165802, + 1.0174933671951294, + 1.0258558988571167, + 1.2219576835632324, + 1.1909079551696777 + ] + ], + "mean_alpha_per_pair": [ + 0.9311889012654623, + 1.2634391784667969, + 0.9381732443968455, + 0.9309640924135844, + 0.9331510066986084, + 1.2567438284556072, + 0.9418283998966217, + 0.9740656316280365, + 1.0187325874964397, + 1.012004017829895, + 0.9977878630161285, + 1.0126410524050395, + 1.0137035648028057, + 1.021284023920695, + 1.0047356883684795, + 1.0267229874928792, + 1.0123908519744873, + 1.015254815419515, + 1.000379502773285, + 1.0117583870887756, + 0.9977450172106425, + 1.0287072857220967, + 1.02439683675766, + 0.9952176411946615, + 1.013523797194163, + 1.010244329770406, + 1.0071445504824321, + 1.0037147402763367, + 1.0029155611991882, + 1.004309316476186, + 1.0369929472605388, + 1.0318179925282795 + ], + "dead_avg_alpha": 1.0159912506739297, + "alive_avg_alpha": 1.0117645418202437, + "ratio_dead_alive": 1.0041775617537276, + "loss_history": [ + { + "step": 290, + "gamma_obs": 0.7363248467445374, + "loss_gamma": 0.055849432945251465, + "loss_ppl": 21.873077392578125 + }, + { + "step": 291, + "gamma_obs": 0.8317559361457825, + "loss_gamma": 0.11006200313568115, + "loss_ppl": 26.562274932861328 + }, + { + "step": 292, + "gamma_obs": 0.7624273896217346, + "loss_gamma": 0.06886813789606094, + "loss_ppl": 17.20162582397461 + }, + { + "step": 293, + "gamma_obs": 0.7731462121009827, + "loss_gamma": 0.07460885494947433, + "loss_ppl": 16.396806716918945 + }, + { + "step": 294, + "gamma_obs": 0.7259001135826111, + "loss_gamma": 0.05103086307644844, + "loss_ppl": 17.004032135009766 + }, + { + "step": 295, + "gamma_obs": 0.746425986289978, + "loss_gamma": 0.06072576716542244, + "loss_ppl": 27.835309982299805 + }, + { + "step": 296, + "gamma_obs": 0.7712214589118958, + "loss_gamma": 0.07356107980012894, + "loss_ppl": 19.29431915283203 + }, + { + "step": 297, + "gamma_obs": 0.6699090600013733, + "loss_gamma": 0.028869088739156723, + "loss_ppl": 22.444265365600586 + }, + { + "step": 298, + "gamma_obs": 0.7840176224708557, + "loss_gamma": 0.08066601306200027, + "loss_ppl": 18.658720016479492 + }, + { + "step": 299, + "gamma_obs": 0.8944492936134338, + "loss_gamma": 0.1555902510881424, + "loss_ppl": 22.22308349609375 + } + ] + }, + "0.65": { + "gamma_after": 0.6972721224187436, + "r2_after": 0.993425061456584, + "gamma_delta": -0.05208684010935305, + "alpha_per_layer": [ + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 0.5888485312461853, + 2.3271522521972656, + 0.6592528820037842, + 0.5935737490653992, + 0.6084148287773132, + 2.39315128326416, + 0.7159332633018494, + 0.774112343788147, + 1.05126953125, + 1.0243884325027466, + 0.929186224937439, + 0.9765675663948059, + 0.9406648278236389, + 0.8773418068885803, + 0.914222002029419, + 1.0652357339859009, + 1.0605310201644897, + 1.091649055480957, + 0.9926859736442566, + 1.0471584796905518, + 0.9154945611953735, + 1.0538314580917358, + 1.195898175239563, + 0.9626159071922302, + 1.1233583688735962, + 0.8999897241592407, + 1.029083490371704, + 0.9848154783248901, + 0.8763991594314575, + 0.944465160369873, + 1.2482807636260986, + 1.2908624410629272 + ] + ], + "mean_alpha_per_pair": [ + 0.9314747552076975, + 1.2211920420328777, + 0.9432088136672974, + 0.9322622915108999, + 0.9347358047962189, + 1.2321918805440266, + 0.9526555438836416, + 0.9623520572980245, + 1.008544921875, + 1.0040647387504578, + 0.9881977041562399, + 0.9960945943991343, + 0.9901108046372732, + 0.9795569678147634, + 0.9857036670049032, + 1.0108726223309834, + 1.0100885033607483, + 1.0152748425801594, + 0.9987809956073761, + 1.007859746615092, + 0.9859157601992289, + 1.008971909681956, + 1.0326496958732605, + 0.9937693178653717, + 1.0205597281455994, + 0.9833316206932068, + 1.004847248395284, + 0.9974692463874817, + 0.9793998599052429, + 0.9907441933949789, + 1.0413801272710164, + 1.0484770735104878 + ], + "dead_avg_alpha": 1.0057722027750984, + "alive_avg_alpha": 1.006664268396519, + "ratio_dead_alive": 0.99911383998675, + "loss_history": [ + { + "step": 290, + "gamma_obs": 0.7522047162055969, + "loss_gamma": 0.010445808991789818, + "loss_ppl": 21.8455867767334 + }, + { + "step": 291, + "gamma_obs": 0.6949314475059509, + "loss_gamma": 0.0020188370253890753, + "loss_ppl": 26.556196212768555 + }, + { + "step": 292, + "gamma_obs": 0.6935656070709229, + "loss_gamma": 0.0018979641608893871, + "loss_ppl": 17.3034725189209 + }, + { + "step": 293, + "gamma_obs": 0.8285958766937256, + "loss_gamma": 0.03189649432897568, + "loss_ppl": 16.508405685424805 + }, + { + "step": 294, + "gamma_obs": 0.7752242088317871, + "loss_gamma": 0.015681108459830284, + "loss_ppl": 16.9216251373291 + }, + { + "step": 295, + "gamma_obs": 0.7267627120018005, + "loss_gamma": 0.005892517510801554, + "loss_ppl": 27.98338508605957 + }, + { + "step": 296, + "gamma_obs": 0.6979865431785583, + "loss_gamma": 0.0023027106653898954, + "loss_ppl": 19.407609939575195 + }, + { + "step": 297, + "gamma_obs": 0.7150024771690369, + "loss_gamma": 0.004225325305014849, + "loss_ppl": 22.519695281982422 + }, + { + "step": 298, + "gamma_obs": 0.7263326048851013, + "loss_gamma": 0.0058266702108085155, + "loss_ppl": 18.817766189575195 + }, + { + "step": 299, + "gamma_obs": 0.7861154675483704, + "loss_gamma": 0.018527427688241005, + "loss_ppl": 22.203859329223633 + } + ] + }, + "0.9": { + "gamma_after": 0.7689464688106303, + "r2_after": 0.985045558920884, + "gamma_delta": 0.019587506282533695, + "alpha_per_layer": [ + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 2.032773733139038, + 0.6663967967033386, + 1.3777823448181152, + 1.2683773040771484, + 1.942412257194519, + 0.7196227312088013, + 1.3388662338256836, + 1.0444575548171997, + 1.162846565246582, + 1.0227490663528442, + 0.9033578634262085, + 1.0714887380599976, + 1.0979971885681152, + 0.8105136752128601, + 0.8913537859916687, + 1.020356297492981, + 1.0055222511291504, + 0.9615305662155151, + 0.9462417364120483, + 0.9389593005180359, + 0.9341173768043518, + 1.035313367843628, + 1.368100881576538, + 0.9937624931335449, + 1.2231770753860474, + 0.8785685300827026, + 1.00326406955719, + 1.0607597827911377, + 0.9275334477424622, + 0.9627155065536499, + 1.2982524633407593, + 1.3724004030227661 + ] + ], + "mean_alpha_per_pair": [ + 1.172128955523173, + 0.9443994661172231, + 1.0629637241363525, + 1.0447295506795247, + 1.15706870953242, + 0.9532704552014669, + 1.056477705637614, + 1.0074095924695332, + 1.0271410942077637, + 1.0037915110588074, + 0.9838929772377014, + 1.0119147896766663, + 1.0163328647613525, + 0.96841894586881, + 0.9818922976652781, + 1.00339271624883, + 1.0009203751881917, + 0.9935884277025858, + 0.9910402894020081, + 0.9898265500863394, + 0.9890195628007253, + 1.0058855613072712, + 1.061350146929423, + 0.9989604155222574, + 1.0371961792310078, + 0.9797614216804504, + 1.000544011592865, + 1.0101266304651897, + 0.9879222412904104, + 0.9937859177589417, + 1.0497087438901265, + 1.0620667338371277 + ], + "dead_avg_alpha": 1.0185589682364806, + "alive_avg_alpha": 1.0133413661409305, + "ratio_dead_alive": 1.0051489086204188, + "loss_history": [ + { + "step": 290, + "gamma_obs": 0.7355005741119385, + "loss_gamma": 0.027060052379965782, + "loss_ppl": 21.920202255249023 + }, + { + "step": 291, + "gamma_obs": 0.8559226989746094, + "loss_gamma": 0.0019428064115345478, + "loss_ppl": 26.550790786743164 + }, + { + "step": 292, + "gamma_obs": 0.752450168132782, + "loss_gamma": 0.02177094668149948, + "loss_ppl": 17.129247665405273 + }, + { + "step": 293, + "gamma_obs": 0.9875119924545288, + "loss_gamma": 0.007658353075385094, + "loss_ppl": 16.369497299194336 + }, + { + "step": 294, + "gamma_obs": 0.7888638377189636, + "loss_gamma": 0.01235124096274376, + "loss_ppl": 16.855716705322266 + }, + { + "step": 295, + "gamma_obs": 0.7187114357948303, + "loss_gamma": 0.03286553546786308, + "loss_ppl": 27.793895721435547 + }, + { + "step": 296, + "gamma_obs": 0.8055550456047058, + "loss_gamma": 0.008919845335185528, + "loss_ppl": 19.270023345947266 + }, + { + "step": 297, + "gamma_obs": 0.8078466057777405, + "loss_gamma": 0.008492243476212025, + "loss_ppl": 22.455759048461914 + }, + { + "step": 298, + "gamma_obs": 0.799404501914978, + "loss_gamma": 0.010119449347257614, + "loss_ppl": 18.785749435424805 + }, + { + "step": 299, + "gamma_obs": 0.8909921050071716, + "loss_gamma": 8.114174124784768e-05, + "loss_ppl": 22.22020149230957 + } + ] + }, + "1.0": { + "gamma_after": 0.7673686331576112, + "r2_after": 0.9821378128353957, + "gamma_delta": 0.01800967062951453, + "alpha_per_layer": [ + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0, + 1.0 + ], + [ + 2.4323229789733887, + 0.5952386260032654, + 1.4820419549942017, + 1.6333458423614502, + 2.346658945083618, + 0.6272574067115784, + 1.5375525951385498, + 1.2864928245544434, + 1.092909574508667, + 1.0609021186828613, + 0.9439433813095093, + 1.0827230215072632, + 1.0821971893310547, + 0.8420082926750183, + 0.9179068207740784, + 0.9373764991760254, + 1.0510554313659668, + 0.9765021204948425, + 0.9544751048088074, + 0.9691928625106812, + 0.9026970267295837, + 1.0211846828460693, + 1.1122674942016602, + 0.9358194470405579, + 1.067273497581482, + 0.9463424682617188, + 0.9989645481109619, + 1.046397089958191, + 0.9529535174369812, + 0.9948934316635132, + 1.1124058961868286, + 1.11370849609375 + ] + ], + "mean_alpha_per_pair": [ + 1.2387204964955647, + 0.9325397710005442, + 1.080340325832367, + 1.105557640393575, + 1.2244431575139363, + 0.9378762344519297, + 1.0895920991897583, + 1.0477488040924072, + 1.0154849290847778, + 1.0101503531138103, + 0.9906572302182516, + 1.0137871702512105, + 1.0136995315551758, + 0.9736680487791697, + 0.9863178034623464, + 0.9895627498626709, + 1.0085092385609944, + 0.9960836867491404, + 0.9924125174681345, + 0.9948654770851135, + 0.9837828377882639, + 1.003530780474345, + 1.01871124903361, + 0.9893032411734263, + 1.0112122495969136, + 0.9910570780436198, + 0.9998274246851603, + 1.0077328483263652, + 0.9921589195728302, + 0.9991489052772522, + 1.0187343160311382, + 1.018951416015625 + ], + "dead_avg_alpha": 1.028175744889439, + "alive_avg_alpha": 1.0031251554135925, + "ratio_dead_alive": 1.0249725463873132, + "loss_history": [ + { + "step": 290, + "gamma_obs": 0.7550089955329895, + "loss_gamma": 0.06002059206366539, + "loss_ppl": 21.867298126220703 + }, + { + "step": 291, + "gamma_obs": 0.8593094348907471, + "loss_gamma": 0.019793834537267685, + "loss_ppl": 26.520620346069336 + }, + { + "step": 292, + "gamma_obs": 0.7391350269317627, + "loss_gamma": 0.06805053353309631, + "loss_ppl": 17.255603790283203 + }, + { + "step": 293, + "gamma_obs": 0.806459903717041, + "loss_gamma": 0.03745776787400246, + "loss_ppl": 16.46477508544922 + }, + { + "step": 294, + "gamma_obs": 0.7993127703666687, + "loss_gamma": 0.040275365114212036, + "loss_ppl": 16.820741653442383 + }, + { + "step": 295, + "gamma_obs": 0.881190836429596, + "loss_gamma": 0.014115617610514164, + "loss_ppl": 27.85260581970215 + }, + { + "step": 296, + "gamma_obs": 0.8282440900802612, + "loss_gamma": 0.029500093311071396, + "loss_ppl": 19.23470115661621 + }, + { + "step": 297, + "gamma_obs": 0.8246332406997681, + "loss_gamma": 0.030753500759601593, + "loss_ppl": 22.452911376953125 + }, + { + "step": 298, + "gamma_obs": 0.8387228846549988, + "loss_gamma": 0.026010308414697647, + "loss_ppl": 18.841869354248047 + }, + { + "step": 299, + "gamma_obs": 0.9339637160301208, + "loss_gamma": 0.004360790830105543, + "loss_ppl": 22.21466827392578 + } + ] + } + } +} \ No newline at end of file diff --git a/data/lerch_validation/lerch_validation.json b/data/lerch_validation/lerch_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..87bcd28b85264645c229bab2cbf89e50e9fb29b5 --- /dev/null +++ b/data/lerch_validation/lerch_validation.json @@ -0,0 +1,167 @@ +{ + "pythia-70m": { + "d_head": 64, + "theta": 10000, + "T_train": 2048, + "z": 0.2896309375740099, + "k_dead": 23, + "gamma_obs": 0.748, + "gamma_pade": 0.752, + "gamma_lerch_dead": 0.4381262073778783, + "r2_lerch_dead": 0.8412032778810773, + "gamma_lerch_full": 0.4823672477284934, + "r2_lerch_full": 0.6235956597226463, + "a_dead": [ + 0.3766197137027542, + 0.3087485064208156, + 0.20871394207869964, + 0.2779640970654833, + 0.07861013207916605, + -0.0652714083843367, + 0.055083187967645754, + -0.08800852998427698 + ], + "a_full": [ + 0.6578634007689929, + 0.589973255408347, + 0.4898061548597748, + 0.5585833995333093, + 0.3573445501084111, + 0.20056785314106362, + 0.2789364609349515, + 0.020115337196340802 + ] + }, + "pythia-1b": { + "d_head": 256, + "theta": 10000, + "T_train": 2048, + "z": 0.2896309375740099, + "k_dead": 91, + "gamma_obs": 0.931, + "gamma_pade": 0.752, + "gamma_lerch_dead": 0.7092055512350871, + "r2_lerch_dead": 0.792455852602181, + "gamma_lerch_full": 0.40966876171772443, + "r2_lerch_full": 0.7971495893077316, + "a_dead": [ + 0.38640983839148113, + 0.324660301068217, + 0.2238293082257104, + 0.16771571478464226, + 0.034844373585812646, + -0.0055807381358166635, + -0.042211915936972565, + -0.07506427311704396 + ], + "a_full": [ + 0.6754663829277784, + 0.6136989798719965, + 0.5127429545592851, + 0.4561832114968159, + 0.32153343043941357, + 0.2689318328181211, + 0.19252351452455274, + 0.048307006654024036 + ] + }, + "Falcon-7B": { + "d_head": 64, + "theta": 10000, + "T_train": 2048, + "z": 0.2896309375740099, + "k_dead": 23, + "gamma_obs": 0.924, + "gamma_pade": 0.752, + "gamma_lerch_dead": 0.4381262073778783, + "r2_lerch_dead": 0.8412032778810773, + "gamma_lerch_full": 0.4823672477284934, + "r2_lerch_full": 0.6235956597226463, + "a_dead": [ + 0.3766197137027542, + 0.3087485064208156, + 0.20871394207869964, + 0.2779640970654833, + 0.07861013207916605, + -0.0652714083843367, + 0.055083187967645754, + -0.08800852998427698 + ], + "a_full": [ + 0.6578634007689929, + 0.589973255408347, + 0.4898061548597748, + 0.5585833995333093, + 0.3573445501084111, + 0.20056785314106362, + 0.2789364609349515, + 0.020115337196340802 + ] + }, + "SmolLM2-360M": { + "d_head": 64, + "theta": 130000, + "T_train": 2048, + "z": 0.022279302890308453, + "k_dead": 18, + "gamma_obs": 0.969, + "gamma_pade": 0.972, + "gamma_lerch_dead": 2.0924869738893457, + "r2_lerch_dead": 0.5372763311992433, + "gamma_lerch_full": 0.17817986024634686, + "r2_lerch_full": 0.8733320313365489, + "a_dead": [ + 0.2904297890018149, + 0.24970981490544591, + 0.1712031729222, + 0.09516771290287208, + 0.023524094760583286, + 0.04453230763710146, + 4.3358119360993597e-07, + -0.0548957162418703 + ], + "a_full": [ + 0.7279244946022235, + 0.6871886379389914, + 0.6085708445246009, + 0.5321387940157193, + 0.45891472241145476, + 0.4691211453886454, + 0.389561520051001, + 0.23980195213522673 + ] + }, + "Qwen2.5-7B": { + "d_head": 128, + "theta": 1000000, + "T_train": 4096, + "z": 0.005792618751480198, + "k_dead": 34, + "gamma_obs": 0.997, + "gamma_pade": 0.997, + "gamma_lerch_dead": 0.5727285650619189, + "r2_lerch_dead": 0.9485129639683372, + "gamma_lerch_full": 0.14150710663250482, + "r2_lerch_full": 0.8935290085544552, + "a_dead": [ + 0.30672222208235267, + 0.26433967484457077, + 0.20591093916598605, + 0.11764468097526831, + 0.06483786379925566, + 0.029049246293096326, + 0.03156951884771056, + -0.11081347319924957 + ], + "a_full": [ + 0.7754712824598464, + 0.7330859163785685, + 0.6746374498044598, + 0.586300738541679, + 0.5332123341643275, + 0.4954626622435493, + 0.49112124240955124, + 0.32341498580283495 + ] + } +} \ No newline at end of file diff --git a/data/master_gamma_results.json b/data/master_gamma_results.json new file mode 100644 index 0000000000000000000000000000000000000000..311537fad701d422844306183a82542566735c83 --- /dev/null +++ b/data/master_gamma_results.json @@ -0,0 +1,393 @@ +[ + { + "model": "EleutherAI/pythia-1.4b", + "corpus": "mongo", + "theta": 10000, + "gamma_obs": 0.7051, + "R2": 0.8413, + "T_attn": 1.4183, + "U": 4.941, + "S": 3.526, + "F": -0.042, + "Cv": 2.255, + "chi": 3.39, + "D90": 1440, + "phase": "A (deconf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "EleutherAI/pythia-14m", + "corpus": "random", + "theta": 10000, + "gamma_obs": 1.0037, + "R2": 0.9777, + "T_attn": 0.9963, + "U": 3.337, + "S": 3.841, + "F": -0.492, + "Cv": 5.776, + "chi": 269.24, + "D90": 836, + "phase": "* Hagedorn", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "EleutherAI/pythia-160m", + "corpus": "random", + "theta": 10000, + "gamma_obs": 1.0171, + "R2": 0.9817, + "T_attn": 0.9831, + "U": 3.26, + "S": 3.585, + "F": -0.268, + "Cv": 5.908, + "chi": 58.33, + "D90": 799, + "phase": "* Hagedorn", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "EleutherAI/pythia-1b", + "corpus": "mongo", + "theta": 10000, + "gamma_obs": 0.9311, + "R2": 0.9831, + "T_attn": 1.074, + "U": 3.755, + "S": 3.542, + "F": -0.046, + "Cv": 4.966, + "chi": 14.52, + "D90": 1028, + "phase": "A (deconf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "EleutherAI/pythia-2.8b", + "corpus": "mongo", + "theta": 10000, + "gamma_obs": 0.6742, + "R2": 0.9993, + "T_attn": 1.4833, + "U": 5.077, + "S": 3.783, + "F": -0.36, + "Cv": 1.953, + "chi": 3.07, + "D90": 1476, + "phase": "A (deconf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "EleutherAI/pythia-31m", + "corpus": "mongo", + "theta": 10000, + "gamma_obs": 1.235, + "R2": 0.9737, + "T_attn": 0.8097, + "U": 2.125, + "S": 3.29, + "F": -0.665, + "Cv": 6.861, + "chi": 4.26, + "D90": 246, + "phase": "B (conf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "EleutherAI/pythia-410m", + "corpus": "mongo", + "theta": 10000, + "gamma_obs": 1.0219, + "R2": 0.9816, + "T_attn": 0.9786, + "U": 3.234, + "S": 3.617, + "F": -0.313, + "Cv": 5.952, + "chi": 45.76, + "D90": 785, + "phase": "* Hagedorn", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "EleutherAI/pythia-70m", + "corpus": "mongo", + "theta": 10000, + "gamma_obs": 0.7476, + "R2": 0.9843, + "T_attn": 1.3376, + "U": 4.741, + "S": 4.332, + "F": -0.787, + "Cv": 2.711, + "chi": 3.96, + "D90": 1383, + "phase": "A (deconf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "google/gemma-2-9b-it", + "corpus": "mongo", + "theta": 10000, + "gamma_obs": 0.6276, + "R2": 0.9773, + "T_attn": 1.5933, + "U": 5.269, + "S": 4.223, + "F": -0.916, + "Cv": 1.55, + "chi": 2.69, + "D90": 1524, + "phase": "A (deconf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "google/gemma-2-9b-it", + "corpus": "random", + "theta": 10000, + "gamma_obs": 1.1348, + "R2": 0.9765, + "T_attn": 0.8812, + "U": 2.613, + "S": 3.751, + "F": -0.786, + "Cv": 6.714, + "chi": 7.42, + "D90": 469, + "phase": "B (conf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "mongo", + "theta": 10000, + "gamma_obs": 0.2871, + "R2": 0.8149, + "T_attn": 3.4836, + "U": 6.212, + "S": 2.466, + "F": -0.683, + "Cv": 0.15, + "chi": 1.4, + "D90": 1726, + "phase": "A (deconf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "random", + "theta": 10000, + "gamma_obs": 0.8266, + "R2": 0.9936, + "T_attn": 1.2097, + "U": 4.337, + "S": 3.472, + "F": 0.113, + "Cv": 3.657, + "chi": 5.77, + "D90": 1254, + "phase": "A (deconf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "meta-llama/Meta-Llama-3-8B", + "corpus": "mongo", + "theta": 500000, + "gamma_obs": 1.0455, + "R2": 0.9975, + "T_attn": 0.9565, + "U": 3.1, + "S": 2.812, + "F": 0.429, + "Cv": 6.161, + "chi": 21.99, + "D90": 718, + "phase": "* Hagedorn", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "mistralai/Mistral-7B-v0.1", + "corpus": "mongo", + "theta": 10000, + "gamma_obs": 1.0608, + "R2": 0.9987, + "T_attn": 0.9427, + "U": 3.014, + "S": 3.011, + "F": 0.186, + "Cv": 6.283, + "chi": 16.46, + "D90": 674, + "phase": "B (conf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "mistralai/Mistral-7B-v0.1", + "corpus": "random", + "theta": 10000, + "gamma_obs": 0.8296, + "R2": 0.9969, + "T_attn": 1.2054, + "U": 4.321, + "S": 4.016, + "F": -0.431, + "Cv": 3.695, + "chi": 5.87, + "D90": 1248, + "phase": "A (deconf)", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "Qwen/Qwen2.5-7B", + "corpus": "mongo", + "theta": 1000000, + "gamma_obs": 0.9967, + "R2": 0.9939, + "T_attn": 1.0033, + "U": 3.378, + "S": 3.371, + "F": -0.005, + "Cv": 5.705, + "chi": 302.61, + "D90": 856, + "phase": "* Hagedorn", + "source": "e_thermodynamics", + "notes": "" + }, + { + "model": "HuggingFaceTB/SmolLM2-135M", + "corpus": "random", + "theta": 10000, + "gamma_obs": 0.9019, + "R2": 0.9995, + "T_attn": 1.1088, + "U": null, + "S": null, + "F": null, + "Cv": null, + "chi": null, + "D90": null, + "phase": "A (near H)", + "source": "e_rlhf_real_pairs", + "notes": "from RLHF pair measurement" + }, + { + "model": "Qwen/Qwen2.5-0.5B", + "corpus": "random", + "theta": 1000000, + "gamma_obs": 0.7992, + "R2": 0.9544, + "T_attn": 1.2513, + "U": null, + "S": null, + "F": null, + "Cv": null, + "chi": null, + "D90": null, + "phase": "A (near H)", + "source": "e_rlhf_real_pairs", + "notes": "from RLHF pair measurement" + }, + { + "model": "HuggingFaceTB/SmolLM2-360M", + "corpus": "random", + "theta": 10000, + "gamma_obs": 0.9192, + "R2": 0.9916, + "T_attn": 1.0879, + "U": null, + "S": null, + "F": null, + "Cv": null, + "chi": null, + "D90": null, + "phase": "A (near H)", + "source": "e_rlhf_real_pairs", + "notes": "smollm2-360m base gamma" + }, + { + "model": "meta-llama/Meta-Llama-3-8B", + "corpus": "random", + "theta": 500000, + "gamma_obs": 0.7589, + "R2": 0.9843, + "T_attn": 1.3177, + "U": null, + "S": null, + "F": null, + "Cv": null, + "chi": null, + "D90": null, + "phase": "A", + "source": "e4_gamma_extended", + "notes": "random corpus (e4 extended validation)" + }, + { + "model": "codellama/CodeLlama-13b-Instruct-hf", + "corpus": "mongo", + "theta": null, + "gamma_obs": 0.5087, + "R2": 0.9761, + "T_attn": 1.9658, + "U": null, + "S": null, + "F": null, + "Cv": null, + "chi": null, + "D90": null, + "phase": "?", + "source": "e4_gamma_partial", + "notes": "partial" + }, + { + "model": "tiiuae/falcon-7b", + "corpus": "mongo", + "theta": null, + "gamma_obs": 0.9247, + "R2": 0.9897, + "T_attn": 1.0814, + "U": null, + "S": null, + "F": null, + "Cv": null, + "chi": null, + "D90": null, + "phase": "?", + "source": "e4_gamma_partial", + "notes": "partial_d1000" + }, + { + "model": "mistralai/Mistral-Nemo-Instruct-2407", + "corpus": "mongo", + "theta": null, + "gamma_obs": 0.376, + "R2": 0.9215, + "T_attn": 2.6596, + "U": null, + "S": null, + "F": null, + "Cv": null, + "chi": null, + "D90": null, + "phase": "?", + "source": "e4_gamma_partial", + "notes": "partial_d1000" + } +] \ No newline at end of file diff --git a/data/mi1_patching/gpt2-large.json b/data/mi1_patching/gpt2-large.json new file mode 100644 index 0000000000000000000000000000000000000000..ea3bb3efb7a6e64944547ab9fd67725eeb122f04 --- /dev/null +++ b/data/mi1_patching/gpt2-large.json @@ -0,0 +1,1876 @@ +{ + "model": "gpt2-large", + "N": 36, + "L_crit": 33, + "N_sem": 3, + "n_heads": 20, + "layers_patched": [ + 31, + 32, + 33, + 34, + 35 + ], + "n_prompts": 10, + "head_summary": { + "31": { + "per_head_R_patch": { + "0": -0.00025166514987412373, + "1": 0.0041670232149281835, + "2": -0.0004417408622913061, + "3": 0.0022094819191501663, + "4": -0.002819798242554766, + "5": -0.004168571639158882, + "6": -0.0008593922200092672, + "7": -0.00073813215703381, + "8": 0.0023619450871236957, + "9": 0.0013771988342524935, + "10": -8.284693949329898e-05, + "11": 0.00028222276321956176, + "12": 0.0007716144055040537, + "13": -0.0029337787896723077, + "14": 0.0006231090890161481, + "15": -8.209510753190098e-05, + "16": 2.8515094671033217e-05, + "17": 0.0005136509087120901, + "18": 0.002893063282768937, + "19": -0.001863910741353117 + }, + "top_head": 1, + "max_R_patch": 0.0041670232149281835, + "n_critical_01": 0 + }, + "32": { + "per_head_R_patch": { + "0": -0.00588079804828694, + "1": 0.0004954645343748178, + "2": -0.0003784957572974985, + "3": -0.0009897141651565012, + "4": -0.0009699988915952744, + "5": 0.007356399831027096, + "6": -1.1955644434390008e-05, + "7": -0.00011028101225841435, + "8": 0.0008336187174203182, + "9": 0.00038803611822342744, + "10": 0.00014096746659893902, + "11": 0.0003024781430083291, + "12": 0.000900868306651719, + "13": 0.0022849127914129795, + "14": -0.0003312000223743564, + "15": 0.0009409494131326428, + "16": 0.0008642989601510276, + "17": -0.0020679459124722894, + "18": 0.0004164571478753263, + "19": 0.00033039139855561785 + }, + "top_head": 5, + "max_R_patch": 0.007356399831027096, + "n_critical_01": 0 + }, + "33": { + "per_head_R_patch": { + "0": 0.003284010290321943, + "1": 0.00033159641184239577, + "2": 0.0005440882465226463, + "3": -0.000665104173763189, + "4": 0.0005099510893382461, + "5": 0.0025110423416743128, + "6": -7.167745792318807e-05, + "7": -0.0003952250008784512, + "8": 0.002837762350041686, + "9": 0.0003689222526274044, + "10": -9.897759351852045e-05, + "11": 0.0014348668356603592, + "12": 0.0010249348390312474, + "13": -0.002967206818586633, + "14": -0.0008872224275506734, + "15": -0.0009352672603965721, + "16": 0.004610879027020559, + "17": -0.0022305161076799905, + "18": 0.0005946148984092451, + "19": -0.0018088293655745102 + }, + "top_head": 16, + "max_R_patch": 0.004610879027020559, + "n_critical_01": 0 + }, + "34": { + "per_head_R_patch": { + "0": -0.000437042417756065, + "1": 0.00037047987548000517, + "2": -0.0009586038981105975, + "3": 0.0008921210702004644, + "4": 0.004799064833414218, + "5": -0.0006341668512759178, + "6": 0.0009805290227763153, + "7": -0.000635139121689893, + "8": -0.0034017797293744378, + "9": 0.0007868529419990467, + "10": -0.0014275434302802401, + "11": -0.004527147638454099, + "12": 0.0004836916781291903, + "13": -0.0010128891319197242, + "14": -0.002472510761653191, + "15": 0.0010183957068256682, + "16": -0.0006182466546547465, + "17": 0.0008559873976733877, + "18": -0.0003291210286053357, + "19": -0.0002306960559252454 + }, + "top_head": 4, + "max_R_patch": 0.004799064833414218, + "n_critical_01": 0 + }, + "35": { + "per_head_R_patch": { + "0": 0.000569630335195493, + "1": 0.0010612978819417962, + "2": -0.00028725740889401356, + "3": 0.00019805984591675014, + "4": 0.0008561672327556585, + "5": 0.0008956377205038909, + "6": 0.0015788886318828757, + "7": 0.0008109569154331359, + "8": 0.0008192536855361551, + "9": -0.0006566973906842708, + "10": 0.004113793941797677, + "11": -0.001331813035183405, + "12": -0.00012806510908374395, + "13": 0.00203613420434474, + "14": 0.0006541638896235466, + "15": 0.0003409894610174412, + "16": 0.0002214590088097079, + "17": 0.0013358728383459063, + "18": -0.00021979710507590307, + "19": 0.0015305769660317768 + }, + "top_head": 10, + "max_R_patch": 0.004113793941797677, + "n_critical_01": 0 + } + }, + "wov_static": { + "31": { + "0": { + "stable_rank_OV": 21.07143139109615, + "frob_norm_OV": 16.747880935668945, + "spec_norm_OV": 3.6484875679016113 + }, + "1": { + "stable_rank_OV": 25.958377771236485, + "frob_norm_OV": 19.46687889099121, + "spec_norm_OV": 3.820828437805176 + }, + "2": { + "stable_rank_OV": 6.865324880227334, + "frob_norm_OV": 16.00592041015625, + "spec_norm_OV": 6.1087188720703125 + }, + "3": { + "stable_rank_OV": 21.276189482943906, + "frob_norm_OV": 11.966377258300781, + "spec_norm_OV": 2.594273567199707 + }, + "4": { + "stable_rank_OV": 21.272344535831206, + "frob_norm_OV": 16.2261905670166, + "spec_norm_OV": 3.5181057453155518 + }, + "5": { + "stable_rank_OV": 10.451239788673204, + "frob_norm_OV": 14.783496856689453, + "spec_norm_OV": 4.572916507720947 + }, + "6": { + "stable_rank_OV": 32.06723692824121, + "frob_norm_OV": 20.148452758789062, + "spec_norm_OV": 3.5580408573150635 + }, + "7": { + "stable_rank_OV": 32.06394527792119, + "frob_norm_OV": 20.91966438293457, + "spec_norm_OV": 3.6944196224212646 + }, + "8": { + "stable_rank_OV": 31.734854102761158, + "frob_norm_OV": 18.536344528198242, + "spec_norm_OV": 3.290454149246216 + }, + "9": { + "stable_rank_OV": 33.087222254003265, + "frob_norm_OV": 24.58754539489746, + "spec_norm_OV": 4.274497032165527 + }, + "10": { + "stable_rank_OV": 27.551843237141277, + "frob_norm_OV": 22.216930389404297, + "spec_norm_OV": 4.232614517211914 + }, + "11": { + "stable_rank_OV": 34.606509079188314, + "frob_norm_OV": 21.598634719848633, + "spec_norm_OV": 3.67153263092041 + }, + "12": { + "stable_rank_OV": 25.195119096749554, + "frob_norm_OV": 17.665477752685547, + "spec_norm_OV": 3.519388198852539 + }, + "13": { + "stable_rank_OV": 4.99081264704517, + "frob_norm_OV": 17.38573455810547, + "spec_norm_OV": 7.782289981842041 + }, + "14": { + "stable_rank_OV": 30.407259463427923, + "frob_norm_OV": 19.527441024780273, + "spec_norm_OV": 3.541250705718994 + }, + "15": { + "stable_rank_OV": 10.515424176236968, + "frob_norm_OV": 16.319456100463867, + "spec_norm_OV": 5.032598495483398 + }, + "16": { + "stable_rank_OV": 29.804701234395598, + "frob_norm_OV": 20.26274299621582, + "spec_norm_OV": 3.711554527282715 + }, + "17": { + "stable_rank_OV": 28.438884709572083, + "frob_norm_OV": 18.28722381591797, + "spec_norm_OV": 3.429189682006836 + }, + "18": { + "stable_rank_OV": 10.668103992214487, + "frob_norm_OV": 16.02682876586914, + "spec_norm_OV": 4.906863689422607 + }, + "19": { + "stable_rank_OV": 11.176052794811492, + "frob_norm_OV": 19.30451202392578, + "spec_norm_OV": 5.774503231048584 + } + }, + "32": { + "0": { + "stable_rank_OV": 21.94971610220422, + "frob_norm_OV": 22.255332946777344, + "spec_norm_OV": 4.750284671783447 + }, + "1": { + "stable_rank_OV": 29.02903273835704, + "frob_norm_OV": 21.65484619140625, + "spec_norm_OV": 4.019192695617676 + }, + "2": { + "stable_rank_OV": 1.6106766023543684, + "frob_norm_OV": 18.098203659057617, + "spec_norm_OV": 14.26038646697998 + }, + "3": { + "stable_rank_OV": 23.380811885712387, + "frob_norm_OV": 19.110050201416016, + "spec_norm_OV": 3.9521374702453613 + }, + "4": { + "stable_rank_OV": 10.410665483568419, + "frob_norm_OV": 16.754491806030273, + "spec_norm_OV": 5.192685604095459 + }, + "5": { + "stable_rank_OV": 22.094452038706244, + "frob_norm_OV": 21.46042823791504, + "spec_norm_OV": 4.565588474273682 + }, + "6": { + "stable_rank_OV": 6.5421949511993205, + "frob_norm_OV": 17.230085372924805, + "spec_norm_OV": 6.736366271972656 + }, + "7": { + "stable_rank_OV": 6.995801969625362, + "frob_norm_OV": 18.008039474487305, + "spec_norm_OV": 6.808441162109375 + }, + "8": { + "stable_rank_OV": 12.445113679631243, + "frob_norm_OV": 17.561542510986328, + "spec_norm_OV": 4.978095531463623 + }, + "9": { + "stable_rank_OV": 7.661732341221949, + "frob_norm_OV": 15.585128784179688, + "spec_norm_OV": 5.630499362945557 + }, + "10": { + "stable_rank_OV": 27.769937859887925, + "frob_norm_OV": 24.00789451599121, + "spec_norm_OV": 4.555820465087891 + }, + "11": { + "stable_rank_OV": 29.117302106620105, + "frob_norm_OV": 20.872074127197266, + "spec_norm_OV": 3.8680317401885986 + }, + "12": { + "stable_rank_OV": 27.081275365362966, + "frob_norm_OV": 28.05231475830078, + "spec_norm_OV": 5.390563011169434 + }, + "13": { + "stable_rank_OV": 27.678946646643382, + "frob_norm_OV": 20.138566970825195, + "spec_norm_OV": 3.8278400897979736 + }, + "14": { + "stable_rank_OV": 28.780630341104075, + "frob_norm_OV": 22.160442352294922, + "spec_norm_OV": 4.130743980407715 + }, + "15": { + "stable_rank_OV": 15.125678314056328, + "frob_norm_OV": 17.97597312927246, + "spec_norm_OV": 4.622053623199463 + }, + "16": { + "stable_rank_OV": 6.672031422624961, + "frob_norm_OV": 20.53367042541504, + "spec_norm_OV": 7.949458122253418 + }, + "17": { + "stable_rank_OV": 13.66326657833936, + "frob_norm_OV": 18.722558975219727, + "spec_norm_OV": 5.065099239349365 + }, + "18": { + "stable_rank_OV": 28.55482275053746, + "frob_norm_OV": 22.008756637573242, + "spec_norm_OV": 4.118658542633057 + }, + "19": { + "stable_rank_OV": 22.434444352843865, + "frob_norm_OV": 20.930208206176758, + "spec_norm_OV": 4.418917655944824 + } + }, + "33": { + "0": { + "stable_rank_OV": 23.36302885887431, + "frob_norm_OV": 16.758277893066406, + "spec_norm_OV": 3.467087745666504 + }, + "1": { + "stable_rank_OV": 20.884097335864006, + "frob_norm_OV": 19.814191818237305, + "spec_norm_OV": 4.335792541503906 + }, + "2": { + "stable_rank_OV": 13.733288568307579, + "frob_norm_OV": 18.093393325805664, + "spec_norm_OV": 4.882392883300781 + }, + "3": { + "stable_rank_OV": 9.266305937126628, + "frob_norm_OV": 14.832252502441406, + "spec_norm_OV": 4.872521877288818 + }, + "4": { + "stable_rank_OV": 25.121449034575075, + "frob_norm_OV": 22.654647827148438, + "spec_norm_OV": 4.51996374130249 + }, + "5": { + "stable_rank_OV": 19.99650773709559, + "frob_norm_OV": 22.042461395263672, + "spec_norm_OV": 4.929274559020996 + }, + "6": { + "stable_rank_OV": 22.997174029927336, + "frob_norm_OV": 18.02042007446289, + "spec_norm_OV": 3.7577481269836426 + }, + "7": { + "stable_rank_OV": 21.60256342659109, + "frob_norm_OV": 20.637351989746094, + "spec_norm_OV": 4.440187454223633 + }, + "8": { + "stable_rank_OV": 31.469190148347835, + "frob_norm_OV": 20.12296485900879, + "spec_norm_OV": 3.5871469974517822 + }, + "9": { + "stable_rank_OV": 24.761072645033767, + "frob_norm_OV": 19.134746551513672, + "spec_norm_OV": 3.8453688621520996 + }, + "10": { + "stable_rank_OV": 21.54440022050514, + "frob_norm_OV": 15.681732177734375, + "spec_norm_OV": 3.3785226345062256 + }, + "11": { + "stable_rank_OV": 1.0798733289759177, + "frob_norm_OV": 32.26720428466797, + "spec_norm_OV": 31.050952911376953 + }, + "12": { + "stable_rank_OV": 11.492980374295932, + "frob_norm_OV": 18.45453643798828, + "spec_norm_OV": 5.443607807159424 + }, + "13": { + "stable_rank_OV": 23.417411943317205, + "frob_norm_OV": 18.537107467651367, + "spec_norm_OV": 3.8306503295898438 + }, + "14": { + "stable_rank_OV": 31.57002754487393, + "frob_norm_OV": 21.109445571899414, + "spec_norm_OV": 3.756983995437622 + }, + "15": { + "stable_rank_OV": 11.711068500730882, + "frob_norm_OV": 17.689620971679688, + "spec_norm_OV": 5.169163227081299 + }, + "16": { + "stable_rank_OV": 34.47294393528205, + "frob_norm_OV": 24.217151641845703, + "spec_norm_OV": 4.124619483947754 + }, + "17": { + "stable_rank_OV": 14.641678852899558, + "frob_norm_OV": 19.733028411865234, + "spec_norm_OV": 5.157013893127441 + }, + "18": { + "stable_rank_OV": 15.257451489677301, + "frob_norm_OV": 17.32320213317871, + "spec_norm_OV": 4.434934139251709 + }, + "19": { + "stable_rank_OV": 32.403603212680935, + "frob_norm_OV": 22.289125442504883, + "spec_norm_OV": 3.9155824184417725 + } + }, + "34": { + "0": { + "stable_rank_OV": 36.754399688217184, + "frob_norm_OV": 16.49030113220215, + "spec_norm_OV": 2.720031499862671 + }, + "1": { + "stable_rank_OV": 22.425161090444465, + "frob_norm_OV": 18.94979476928711, + "spec_norm_OV": 4.001628398895264 + }, + "2": { + "stable_rank_OV": 1.1838441893161384, + "frob_norm_OV": 26.03415298461914, + "spec_norm_OV": 23.92743682861328 + }, + "3": { + "stable_rank_OV": 28.81603566259894, + "frob_norm_OV": 22.185331344604492, + "spec_norm_OV": 4.132842063903809 + }, + "4": { + "stable_rank_OV": 42.28037897310927, + "frob_norm_OV": 24.413969039916992, + "spec_norm_OV": 3.754645586013794 + }, + "5": { + "stable_rank_OV": 23.48870039738928, + "frob_norm_OV": 14.70926284790039, + "spec_norm_OV": 3.0350191593170166 + }, + "6": { + "stable_rank_OV": 40.60502532229171, + "frob_norm_OV": 23.455183029174805, + "spec_norm_OV": 3.680856943130493 + }, + "7": { + "stable_rank_OV": 11.793041610575692, + "frob_norm_OV": 13.951476097106934, + "spec_norm_OV": 4.062629699707031 + }, + "8": { + "stable_rank_OV": 1.531437443099887, + "frob_norm_OV": 18.46465492248535, + "spec_norm_OV": 14.920781135559082 + }, + "9": { + "stable_rank_OV": 31.511287529310795, + "frob_norm_OV": 19.60415267944336, + "spec_norm_OV": 3.492327928543091 + }, + "10": { + "stable_rank_OV": 3.651858234487035, + "frob_norm_OV": 14.5047025680542, + "spec_norm_OV": 7.590176105499268 + }, + "11": { + "stable_rank_OV": 2.0226542576005264, + "frob_norm_OV": 21.490337371826172, + "spec_norm_OV": 15.110624313354492 + }, + "12": { + "stable_rank_OV": 33.21445073810024, + "frob_norm_OV": 15.884390830993652, + "spec_norm_OV": 2.756176471710205 + }, + "13": { + "stable_rank_OV": 37.05581386116528, + "frob_norm_OV": 17.294517517089844, + "spec_norm_OV": 2.8410592079162598 + }, + "14": { + "stable_rank_OV": 17.05855904401088, + "frob_norm_OV": 22.393463134765625, + "spec_norm_OV": 5.421882152557373 + }, + "15": { + "stable_rank_OV": 16.09067636982799, + "frob_norm_OV": 17.53775405883789, + "spec_norm_OV": 4.372067451477051 + }, + "16": { + "stable_rank_OV": 29.504184607519512, + "frob_norm_OV": 17.50582504272461, + "spec_norm_OV": 3.2228550910949707 + }, + "17": { + "stable_rank_OV": 35.072758591977355, + "frob_norm_OV": 14.341248512268066, + "spec_norm_OV": 2.421597719192505 + }, + "18": { + "stable_rank_OV": 33.557705571103526, + "frob_norm_OV": 17.65315055847168, + "spec_norm_OV": 3.0473763942718506 + }, + "19": { + "stable_rank_OV": 35.35101806206843, + "frob_norm_OV": 18.11528778076172, + "spec_norm_OV": 3.046802282333374 + } + }, + "35": { + "0": { + "stable_rank_OV": 11.909864770618327, + "frob_norm_OV": 12.952859878540039, + "spec_norm_OV": 3.753291130065918 + }, + "1": { + "stable_rank_OV": 11.99061104442025, + "frob_norm_OV": 11.987768173217773, + "spec_norm_OV": 3.4619252681732178 + }, + "2": { + "stable_rank_OV": 3.6853344665674115, + "frob_norm_OV": 15.226149559020996, + "spec_norm_OV": 7.931432723999023 + }, + "3": { + "stable_rank_OV": 8.13921295055685, + "frob_norm_OV": 14.903474807739258, + "spec_norm_OV": 5.2239179611206055 + }, + "4": { + "stable_rank_OV": 7.88602862412596, + "frob_norm_OV": 13.213994026184082, + "spec_norm_OV": 4.705491065979004 + }, + "5": { + "stable_rank_OV": 14.659592432760887, + "frob_norm_OV": 16.328350067138672, + "spec_norm_OV": 4.26462984085083 + }, + "6": { + "stable_rank_OV": 11.828318147307364, + "frob_norm_OV": 12.77999496459961, + "spec_norm_OV": 3.715944290161133 + }, + "7": { + "stable_rank_OV": 21.155614893569425, + "frob_norm_OV": 19.040937423706055, + "spec_norm_OV": 4.139763355255127 + }, + "8": { + "stable_rank_OV": 13.600890265029928, + "frob_norm_OV": 14.2173490524292, + "spec_norm_OV": 3.8550939559936523 + }, + "9": { + "stable_rank_OV": 15.474427530838067, + "frob_norm_OV": 15.755125999450684, + "spec_norm_OV": 4.005111217498779 + }, + "10": { + "stable_rank_OV": 17.09303326037806, + "frob_norm_OV": 20.549291610717773, + "spec_norm_OV": 4.970353603363037 + }, + "11": { + "stable_rank_OV": 16.900564404372904, + "frob_norm_OV": 15.440071105957031, + "spec_norm_OV": 3.755767345428467 + }, + "12": { + "stable_rank_OV": 10.615997149422835, + "frob_norm_OV": 13.039316177368164, + "spec_norm_OV": 4.0019755363464355 + }, + "13": { + "stable_rank_OV": 7.74252195089608, + "frob_norm_OV": 13.823925018310547, + "spec_norm_OV": 4.968097686767578 + }, + "14": { + "stable_rank_OV": 8.830676942033529, + "frob_norm_OV": 11.165353775024414, + "spec_norm_OV": 3.7572968006134033 + }, + "15": { + "stable_rank_OV": 6.0034632116858635, + "frob_norm_OV": 12.992203712463379, + "spec_norm_OV": 5.302515029907227 + }, + "16": { + "stable_rank_OV": 9.915275724022662, + "frob_norm_OV": 14.13181209564209, + "spec_norm_OV": 4.487923622131348 + }, + "17": { + "stable_rank_OV": 1.196065776765905, + "frob_norm_OV": 20.76240348815918, + "spec_norm_OV": 18.984540939331055 + }, + "18": { + "stable_rank_OV": 5.314463818503118, + "frob_norm_OV": 13.582500457763672, + "spec_norm_OV": 5.89182710647583 + }, + "19": { + "stable_rank_OV": 20.082230652914788, + "frob_norm_OV": 17.0269775390625, + "spec_norm_OV": 3.7995448112487793 + } + } + }, + "raw_patching": { + "31": { + "0": [ + 0.00010245327371320225, + -0.0001368268373450451, + -0.00021206219062888025, + -0.001307240539233685, + -0.0004255376771123958, + -0.0003244618912956859, + -0.0001260136087080051, + -2.1046884425849473e-05, + 1.905124330173725e-05, + -8.496638700663007e-05 + ], + "1": [ + 0.0005720414906195106, + 0.0008169815306573887, + 0.00974434528840144, + -0.000251372879698809, + 0.0020765294935367945, + 0.0006813358537512983, + -0.0005043161453381638, + 0.0014149472380149827, + 0.027871333908998208, + -0.0007515936296608092 + ], + "2": [ + -0.020193064618868727, + 0.0009006355624005227, + 0.009050557250961058, + 0.0049150253605108945, + 0.003427894109823402, + -0.0007086302294649207, + 0.0012653048695763815, + 0.004564025173920429, + -0.0005429604340995116, + -0.007096195667672589 + ], + "3": [ + -0.007809587559127307, + 0.00461841378083096, + 0.02030936541260302, + 0.0017059839435559169, + -7.822840285404418e-06, + 0.0021472997143451348, + 0.003678733729603476, + 0.004342950025699301, + -0.0028119635113364183, + -0.004078553504387019 + ], + "4": [ + -0.004106358062101045, + -0.0025035247070657757, + -0.0005859459427018372, + -0.0016285915660486474, + -0.002154385380186772, + -0.0022261975191423243, + 0.0009315585466171219, + -0.009240742327443832, + -0.0002545608985936892, + -0.006429234568880859 + ], + "5": [ + -0.015943631909788016, + 0.003190114325896822, + -0.003991209053378815, + -0.0029570591484568984, + -0.001556372700591412, + 0.00047133975060461626, + 0.001631110729538197, + -0.0009906950795096705, + 0.002325975366729721, + -0.023865288672633368 + ], + "6": [ + -0.002706077622254229, + -0.0023031106451872354, + -0.002198319623516464, + 0.0006977501751639668, + -0.0005920524203302898, + 0.0010521981837601424, + 0.0038370685858201805, + -0.0019847046290076636, + -0.0015052296612496404, + -0.0028914445432914396 + ], + "7": [ + -0.000551859352635856, + -0.0006550923518184493, + -0.006095006190824335, + -0.0011540808210171985, + -0.0005895689789698439, + -0.00031866183645654116, + 0.002238213676164147, + 0.0014543894308760234, + -0.0006498288370016378, + -0.0010598263086544091 + ], + "8": [ + 0.0034935666495654926, + 0.0009228191214255239, + 0.00046525159398578576, + 0.0015692777229197163, + 0.004198257619833704, + 0.0008002369779537658, + 0.004748867989223482, + 0.0064076991049249596, + 0.00045668551800450157, + 0.0005567885734000286 + ], + "9": [ + 0.002803517498633912, + 0.003942915941512414, + 0.003112231719989798, + 0.0005249884505709671, + 0.0003905211539301094, + 0.002580000864330154, + -0.0002738800446789768, + -0.0011766368458545767, + 0.0032555853196483, + -0.0013872557155571693 + ], + "10": [ + 0.00016107145792050492, + -0.009702766971259557, + 0.0010462903124416653, + 0.0024381645858786295, + 0.0014160582637262218, + -0.001791363996054678, + 0.0029509168120189235, + 0.002701790210981291, + -0.00044416612954907423, + 0.00039553605896308435 + ], + "11": [ + -0.00016274259036501136, + 0.0003770358333524046, + 0.004794358088300536, + -0.0006063418552734907, + 0.0001991719971077569, + 0.0003456150324737432, + -0.0009887422922094358, + -0.0022601370850372842, + 0.0010278599362318242, + 9.615056761457548e-05 + ], + "12": [ + -0.002389719395644203, + 0.0001806012648867458, + -0.0002808801687448089, + -0.0012755523216716534, + -0.0012177554710946208, + 0.0012755002950672152, + 0.002196994271446575, + -0.001602049068855802, + 0.012800258213818663, + -0.0019712535641675725 + ], + "13": [ + -0.0018954498380190303, + 0.0007942899511967006, + -0.008429910222519951, + -0.0030760930939142737, + -0.0036463127774746143, + 0.003870513066039862, + 0.0040960311316489886, + -0.012101129927366366, + -0.0026583742022419365, + -0.006291351984072457 + ], + "14": [ + 7.430111945574768e-05, + -0.0012335582879207687, + 0.00026849526945739216, + -0.00010674460467210031, + 0.001369866254421929, + -0.002328210248371988, + 0.00204808099345103, + 0.005383859325532062, + 0.00021373680580425222, + 0.0005412642630039253 + ], + "15": [ + 0.002543463580538795, + -8.941159668092056e-05, + -0.0005624613695247545, + -0.00041895479949801497, + -0.0034943261662153287, + -0.00025289944997153226, + 0.0004456929919620617, + 0.0001239611775632709, + -0.00014170496208244565, + 0.0010256895185898594 + ], + "16": [ + 0.0005342481876437498, + 0.0012162009230347946, + -0.0010950120388836726, + 0.00029606139164526393, + 0.0010757026252771185, + 0.003327952053807512, + -0.00013517347642302106, + -0.0016668469571275118, + -0.0017753944353095144, + -0.0014925873269543866 + ], + "17": [ + 9.795407097799261e-05, + -0.002325124864067007, + 0.007637744022815373, + -0.0019714743304378108, + -0.0028375800984454243, + 0.0013445891835923219, + 0.003781847669023942, + 0.0013738478101597805, + -0.003597600497017583, + 0.0016323061205193166 + ], + "18": [ + 0.0032736199101385303, + -0.002039448039065657, + -0.004998475249197866, + -0.0015556680397360231, + 0.014355532784057216, + -0.005902749927767258, + 0.002474276552705415, + 0.019619673648877695, + -0.0009685289213778424, + 0.004672400109055165 + ], + "19": [ + -0.008398597471491036, + 0.0020428348419702373, + -0.004384955379780648, + -0.0038106097266341877, + -0.003582115818307089, + -0.0019791834189340418, + 0.006794135594733472, + -0.0029984352749361774, + -0.0009991923510730195, + -0.001322988409078677 + ] + }, + "32": { + "0": [ + -0.0008283674978768824, + -0.002145285629833792, + -0.0045125432101755444, + 0.00013254206384119018, + 0.00015111740678312976, + -0.013905887361621861, + 0.0006143654131714268, + -8.236457920982038e-05, + -0.038381541957741856, + 0.00014998486979461123 + ], + "1": [ + -0.003392527410997642, + -8.297667116221795e-05, + 0.008286432332661953, + -0.0006862733527777181, + 0.0010459013289517683, + -0.00112282238091914, + 0.0003341388887184746, + -0.002634009299720091, + 0.001085376546961831, + 0.0021214053620309593 + ], + "2": [ + -0.012038323935626052, + 0.00040429959673427625, + 0.001705844618832866, + 0.0010087415923913418, + -0.001375826513686999, + -0.0022806156807213588, + -0.0044303663032470705, + 0.009302060022227802, + 0.0009841327968440272, + 0.0029350962332761817 + ], + "3": [ + -0.0016895149013960065, + -0.00045569433081128264, + -0.00045777391894432664, + 0.0006381316376354008, + 0.00046713531989986374, + -0.0011124164001783217, + -0.00013268722661465959, + -0.0032266365336321985, + 0.0003314009132440294, + -0.004259086210767511 + ], + "4": [ + -0.03148644913019364, + -0.004386333111794593, + 0.0019815838859866715, + 0.0044086232683176576, + -0.006618247053520159, + 0.006958018728795182, + 0.0008501665792065516, + 0.008618284863636819, + -0.0017281292078799662, + 0.011702492261492732 + ], + "5": [ + 0.008284189173367134, + 0.0024488278401568037, + 0.0037694492529305426, + 0.0044707809258431805, + 0.011661868112449628, + -0.00426397855093655, + 0.007382068246784424, + 0.019417159532591176, + 0.01012346780971838, + 0.010270165967366232 + ], + "6": [ + -0.00018973780677626915, + 0.001587225181231569, + 0.00045508662947630225, + -0.0014213993742969017, + -0.0003802148722842591, + -0.00025776126064552124, + -0.0011053343226962815, + -0.0005117541661970328, + 0.0015850634427045393, + 0.00011927010513995517 + ], + "7": [ + 0.0006514845560583551, + 0.0006553463620362928, + -0.0008414721195091987, + 0.0004831437530211048, + -0.000985429531824912, + -0.002535818093643728, + 0.0004999979219867992, + 0.0004308810984819577, + 0.00011430745981042352, + 0.0004247484709987627 + ], + "8": [ + 0.0021411063073614764, + 0.0006892990611547105, + 0.002023178453404788, + 0.002873776012556687, + -0.001794286382922124, + 0.0053125090529601736, + 0.0009772924575656657, + 0.001729821886905644, + 0.0004595885646028615, + -0.006076098239386702 + ], + "9": [ + 0.0034242789274432643, + 0.0003956632493275964, + -0.00014861879144900038, + -0.00013071389744338067, + -0.0009396100387246861, + -0.00021920795495002963, + -0.0003296898101140383, + -0.0007000160615337651, + 0.00035145007881395293, + 0.0021768254808643603 + ], + "10": [ + 0.00037201979187704806, + -0.001279449467277832, + -0.0001081926107561119, + 0.0009345992884801779, + 0.001132449260363306, + 0.0020224279454553124, + 0.0018307304180990088, + 0.00013307597003115848, + -0.0020783092038071365, + -0.0015496767264755408 + ], + "11": [ + 0.00025979682079596203, + -0.0024643224634452584, + 0.005625782081972773, + -0.0004989878618021207, + 0.0008446184066876325, + -0.0020026395230629363, + 0.00367664004555433, + 0.0004537509414013847, + -0.0018505107660420784, + -0.0010193462519763978 + ], + "12": [ + 0.0008493209277580016, + 0.004088633136481982, + -0.0036420950998806915, + 0.0007238523287326915, + 0.001566927326373307, + 0.0020481017176109386, + 0.0004159888495147957, + 0.0012230394256910954, + 0.0013517310723613575, + 0.00038318338187371185 + ], + "13": [ + -0.0007500813702842348, + 0.003277917191198067, + 0.014635445797550852, + -0.0008797542965458923, + -0.0002495858567248076, + -0.0007123832061255439, + -0.0020506326708859273, + -0.00036144295222659604, + 0.010033291924756825, + -9.364664658294591e-05 + ], + "14": [ + 0.0010605263590136996, + -4.064163485496389e-05, + -0.0003619662074756314, + 0.0006652494392029087, + -0.00139917086247519, + -0.0024486466812083467, + 0.00022166879870338593, + -0.0006592480806773953, + 0.0014090662426789666, + -0.0017588375966509976 + ], + "15": [ + 0.001768957966834851, + 0.0025704140644312373, + 0.0044501513590483694, + 0.0014445561486691556, + -0.0003091884493755079, + -0.001444810719415779, + -0.0008489888819289067, + 0.0011461437219620074, + 0.002474393624070874, + -0.0018421347029698744 + ], + "16": [ + -3.3294100240551296e-05, + -0.003542172487827947, + -0.0011134725491422747, + 0.000779306709466247, + 0.00921182903830178, + -0.002863862371751827, + 0.00034349503931309806, + 0.008671150659950567, + -0.0029678208255858687, + 0.00015783048902705054 + ], + "17": [ + 0.0012784163200474233, + -0.0015880718819577143, + -0.005725328630962198, + -0.000295756697245629, + 0.002147555916445546, + -4.1965102659694394e-05, + -0.0003760779973279405, + 0.0006493046707124271, + -0.016949800445349435, + 0.00022226472357431815 + ], + "18": [ + 0.0006931343185214386, + 0.004533743708216451, + -0.003538809713370537, + 0.001524487646173383, + 0.0027578616307751126, + -0.0034039498311851297, + 0.0016663762202410084, + -0.006488572172640035, + 0.007163630362278002, + -0.0007433306902564316 + ], + "19": [ + -0.0006423576019383584, + 0.001852327178587594, + 3.5635795119453704e-05, + -0.0013064280208346585, + 0.0010835254655625229, + 0.001280106220968889, + -0.0011967367169668337, + 0.0017914710286884472, + 0.0009097422277610531, + -0.0005033715913919312 + ] + }, + "33": { + "0": [ + -0.0005778261798504944, + -0.00033156800435841377, + 0.0032553590938302268, + 0.02430100340768292, + -0.00035364204972748855, + 0.0007721749479231978, + 0.005800813368666528, + -0.00026084878808100057, + 0.0005386965844081705, + -0.0003040594772742174 + ], + "1": [ + 0.0007538092811219799, + -0.00047152763439019566, + 0.0023921549812317873, + -2.1633302374079318e-05, + 0.00031527288070860025, + -0.00031900301615296145, + -0.0002833016229001361, + 0.0009466126286649778, + 0.000731386302374313, + -0.0007278063798603283 + ], + "2": [ + 0.0010435579372694804, + 0.0002498613842854134, + -0.00128148656023006, + 0.0014908696974136634, + 0.00024797161984051777, + 0.0007233009564109928, + 0.003210402778860018, + 0.00043021820448429317, + -0.000982046232101456, + 0.00030823267899360003 + ], + "3": [ + -0.006195659263682821, + 0.0014496363132329934, + 0.00035565691915940027, + -0.00015651135661247056, + -0.0003515311245711096, + 0.002451546708627919, + 0.001036242749074447, + -0.0021517539164191304, + 0.0011833543696564796, + -0.0042720231360975965 + ], + "4": [ + -0.0008225828086458986, + 0.002468555967075984, + 0.003012100977637759, + 0.0019638569704469376, + -0.000632035826233468, + -4.332982144537551e-05, + -0.0014319490343631359, + 0.00026714628105881376, + 0.000593763749570811, + -0.00027601556171996623 + ], + "5": [ + 0.0008193690924064631, + -0.00039193776613255803, + 0.005663871489215206, + 0.018206201331785618, + -0.00043149793637746583, + 0.00018099582895095832, + 0.0024848758282042193, + -0.0014273765004711929, + -0.00019414124126532247, + 0.00020006329042720263 + ], + "6": [ + 0.00020464945012724965, + 0.0018448762121975172, + 0.001274943594568783, + 0.0011733781329940768, + 0.0009114229792836257, + -0.0019196475619087032, + -0.004998016391071916, + -2.121260792526561e-05, + -0.0002549237794184842, + 0.0010677553919212362 + ], + "7": [ + 0.001117216313477341, + -0.0008792140340290523, + 0.003591971309368411, + -0.0007110551639480249, + -0.005631203284810957, + 0.0018943149694494975, + -0.002871160535271749, + -0.0037181724329004625, + 0.0019414124126532247, + 0.0013136404372272599 + ], + "8": [ + 0.002325187973556339, + 0.011085175246836632, + 0.0077835586861238265, + 0.002177955568590408, + 0.0005161832867686693, + 0.0007497423828835644, + -0.00020701301035936043, + 0.002207271288723536, + -0.001947127785643746, + 0.003686689862936991 + ], + "9": [ + -0.0006314309667242779, + -0.0010967314505757235, + 0.0020769242427652757, + -0.0001892152221732853, + 0.0005796352135280606, + 0.0010405980740818528, + 0.0006565008046605002, + 0.0003659174867108318, + 0.0007665857423794275, + 0.00012043860162138231 + ], + "10": [ + -3.162296779604486e-05, + 0.000798015434391739, + -0.00018857761745179764, + 0.00032267136921338026, + -0.00023071170238541915, + -0.0007446246874372603, + -7.000756039333617e-05, + -0.0005453960365785087, + -7.058032042262657e-05, + -0.00022894184632533032 + ], + "11": [ + 0.007625248795633289, + -0.0019005891199778636, + 0.0022659692149071646, + -0.002136821824639694, + -0.008152517126003603, + 0.0009065144533886829, + -0.0014341081460388183, + -9.943409964968255e-06, + 0.008743613473435411, + 0.008441302045863993 + ], + "12": [ + -0.00045146285731589245, + 0.0032344814439468246, + 0.0016924081714927441, + 0.0014597908686509017, + 0.0018489220928519328, + 0.0019307359020423623, + -0.0022957245598891397, + 0.002333055424780385, + -0.0009012145283783708, + 0.0013983564321307272 + ], + "13": [ + 0.0004022287245277413, + 0.0017704512183693646, + 0.0034794556677289552, + -0.03742490215355807, + 0.00147988270668968, + 0.001990271759067701, + -0.0036036228143403455, + 0.0012121016747296302, + 0.0014239443564955615, + -0.0004018793255765459 + ], + "14": [ + -8.959840875546043e-05, + 0.001717363082840068, + 0.0013497203449833745, + -0.0002905768924518354, + -0.0005612577474607614, + -0.004839122224177036, + -0.0023563105420613168, + -0.001045383834316996, + -0.0009619970665315325, + -0.0017950609875752386 + ], + "15": [ + -0.008492952180280861, + 0.0011232331833040646, + -1.2384899287416698e-05, + 2.4477116770671904e-05, + -0.0018191207965265826, + 0.0026120717557936604, + -0.0011339261954924386, + -0.0037456825338035415, + 0.0006511896400946191, + 0.0014404223054621038 + ], + "16": [ + 0.002562488780676253, + 0.00031895216353885206, + 0.005714228957072532, + 0.00013833125743425368, + 0.0003401072943130587, + 0.003706064452365287, + 0.02092499809106204, + 0.008692528991375249, + -0.0006226127751420131, + 0.004333703057510072 + ], + "17": [ + 0.003908650239050976, + -0.009050553401910001, + -0.009160151925787445, + -0.00024487273250659735, + 8.41886621191142e-05, + -0.0009794416134985176, + -0.000554891700650359, + 0.0002086458857649172, + -0.007723374034524282, + 0.0012066395451422896 + ], + "18": [ + 7.095885456673481e-05, + -1.9474116701336867e-06, + 0.005495623800782375, + 0.0014378528718771873, + -0.0001391968882529897, + 0.00011395401860437339, + -0.0018106441367525096, + -0.00012230394256910953, + -0.00027624302787519013, + 0.0011780948453817125 + ], + "19": [ + -0.0028369401475240408, + -0.0013842710171745932, + -0.005275733419094468, + -7.089223031505805e-05, + -0.0010601811167743318, + -0.0005499816706294907, + 0.0007529411261743109, + -0.0003687347862009061, + -0.00536773316036757, + -0.0019267672338389539 + ] + }, + "34": { + "0": [ + 0.00019847911494753358, + 0.0008147801087694115, + -0.004524694432117915, + -0.00030215527963796233, + 0.00037822811919590243, + -0.00015711325020153875, + -0.0004070252646793872, + -0.0004527566004048879, + 0.00028849025571202124, + -0.00020665694914382717 + ], + "1": [ + -0.0002780507290359554, + 0.005477561007650373, + -0.004686165477544424, + 0.00019754353576330644, + 8.90313727719836e-05, + 0.0019650244615326004, + 0.00018162709126345907, + 0.00013556182252240053, + -0.0003367534054097556, + 0.0009594190752860635 + ], + "2": [ + -0.011892935412953992, + -0.0016534371780161144, + -0.0029532142866862685, + -0.0010098588051900032, + -0.0034611722240533763, + 0.00014158957401441604, + 0.003320844612452496, + 0.0016774532610901445, + -0.0006011120862729097, + 0.006845803564509631 + ], + "3": [ + 0.00129551329044122, + -0.002406069453486477, + 0.0038758892835988124, + 9.872098548171407e-05, + 0.0003608440296727815, + 0.0020894697558018972, + 0.004609834282834848, + -0.00032912686984044924, + -0.0005485850868838341, + -0.00012527951561586614 + ], + "4": [ + 0.0013830549208034416, + -0.001215946912816951, + 0.005405307506922251, + -0.001430743335885706, + 0.00047384061157306753, + 0.005348929985553038, + 0.029213042682488075, + -6.131769478397091e-06, + 0.007004870001430191, + 0.0018144246435531739 + ], + "5": [ + -0.00014628836321910183, + -0.0021676385290040224, + 0.00324589516135588, + -0.0012844900240609443, + -0.0008086085069611676, + 0.00012998946433612652, + -0.0014034225891935148, + -0.00018362163735308044, + -0.0008095871201176344, + -0.002913896368541718 + ], + "6": [ + 0.0019679512763806943, + 0.0011031663760944263, + 0.0021165325527505046, + -0.0008363861269978553, + -0.0013924655708019863, + 0.0027574143064686995, + 0.004787143150746942, + -0.0019532171641185973, + -6.549998887549664e-05, + 0.0013206514161158227 + ], + "7": [ + -0.00297191622958033, + 0.0001137119075212844, + 0.0001343644734012189, + 6.652494392029086e-05, + -0.0004067876948410297, + -7.079478700720801e-06, + -0.0006657261000020519, + -0.00017152382189570238, + -0.00023333237034318195, + -0.0022096268463787077 + ], + "8": [ + -0.009271057156172973, + 0.0006306227008328564, + 0.006279844970755403, + -0.0012837790704617962, + -0.00880230955796426, + 0.003610619432291714, + -0.012332322470485185, + -0.013370240485895147, + 0.004944251237831811, + -0.004423426894476798 + ], + "9": [ + 1.4140351453515994e-05, + -0.0009545703986559644, + 0.004945313653199992, + 0.0008698009461578182, + -0.0004321187967175773, + 0.0009454942337046999, + 0.0004818090418098389, + 0.0016819277955743802, + -0.001530994199810085, + 0.0018477267932738473 + ], + "10": [ + -0.00962418029656214, + 0.0003580697370867548, + 0.004801485247324426, + -0.000785908421458337, + -0.0018874154339388436, + 0.00125690600161231, + -0.005407985898945416, + -0.005087214261577175, + -6.549998887549664e-05, + 0.002166309012531516 + ], + "11": [ + -0.02485321026334929, + -7.08688507783433e-05, + 0.0019334463529072784, + -0.004867797728567482, + -0.0029251214064011403, + 0.00020675489603068944, + -0.0018553966333030162, + -0.009857730915770111, + 0.004123324091939332, + -0.0071048759272489045 + ], + "12": [ + 0.000584125063679788, + 0.00016332857007338614, + 0.00043907973134067876, + 0.000810791797428522, + -0.00011249989362819685, + -0.0004785045242294421, + 0.0012759695727017216, + 0.0001239611775632709, + 0.0016861257524099455, + 0.00034453953395222883 + ], + "13": [ + -0.0030207647164197486, + -0.0010497395602746715, + -0.0010681391442034288, + 0.0001924652957693911, + -0.0022174647907420965, + -0.0011891818318728845, + -0.0015502421831399135, + -0.0003647574222149188, + -9.852214393184122e-05, + 0.00023745517783287087 + ], + "14": [ + -0.0011564236515984535, + -0.001549038978482426, + -0.003004155947906209, + -8.886919989351833e-05, + 0.00017135745387076342, + -0.012729073293744212, + -0.0007865709262137266, + -0.0013402059397783046, + -0.0038524335562298687, + -0.0003896935765559487 + ], + "15": [ + -0.0012422941495161689, + 3.3868029045803244e-07, + -0.0031264860380375795, + 0.00910782342908715, + 0.002194741302294017, + 0.002128449536117914, + -0.0031333944624273484, + 0.0028996640692841593, + -0.0004133212594414996, + 0.0017684359606055776 + ], + "16": [ + -0.004171917873385528, + -0.0018176124488156456, + 0.0024945056583617594, + 0.00047400292103205716, + -0.000935760704615995, + -0.0019454919239125393, + 0.001082696363914885, + -0.00047065473834183075, + -0.001488083542278077, + 0.0005958497414934499 + ], + "17": [ + 0.0016212555684703975, + 0.0012029077216343168, + -0.002194230269978166, + 0.0012270043473298227, + 0.0009027309345220652, + 0.0001153187373900545, + 0.004256197961408696, + 0.000881649016893852, + 8.47326725896314e-05, + 0.00046230728647320623 + ], + "18": [ + -0.000609963342244849, + 0.000380422636256985, + 0.00015025453286431957, + -0.0002564511196927243, + -0.00022723488448079496, + -0.0007644984047537415, + -0.00020570445782864386, + -0.0021890417037877612, + 0.0013627989375176047, + -0.0009317924799037506 + ], + "19": [ + 0.0007946877516875989, + -0.003414828698615727, + -0.0043508384874039905, + -0.0003413592923909887, + 0.001032614917673383, + -0.0012685061112905995, + 0.0033924224358826923, + 0.0014772592737954504, + 0.0007148752248461406, + -0.00034328757343641403 + ] + }, + "35": { + "0": [ + 0.005262524616400343, + -0.001172087815202636, + -0.0025436947394938576, + -0.0010484534291437596, + 0.0003954880366510011, + -0.005142601564142874, + 0.003940771373879468, + 0.003756786008264423, + -0.001498879246815728, + 0.00374645011155855 + ], + "1": [ + 0.00384720398455297, + 0.002887334146227341, + 0.002229982903770142, + 0.0008989500437228923, + 0.0018307929709206782, + -0.0013318802399006665, + 0.0013247785820974494, + 0.0005641227920125323, + 0.00030899302231293847, + -0.0019472993862983165 + ], + "2": [ + -0.001164907862470563, + -0.0020888953614725298, + 0.0028590423166328922, + -0.0009887333268153154, + -0.004017463088793246, + 0.0060637867444776295, + -0.0004137643102125775, + 0.0012233708726899276, + -0.006321021087104022, + 0.001976011014127669 + ], + "3": [ + -0.01142309009874853, + 0.006735927626847193, + 0.0071458532114943895, + 0.0005798334425052527, + -0.002178723105519141, + 0.004436615477325211, + -0.0038653333204836586, + -0.0011123361280811154, + 0.002994855447033096, + -0.0013330040932051953 + ], + "4": [ + -0.0013554169611442969, + 0.0018698538836187972, + 0.001745336090145572, + -0.001134986638640077, + 0.001251406101528662, + 0.000602864523574634, + 0.00468193552727733, + 0.001469635992822308, + -0.001047455500770754, + 0.00047849930914441077 + ], + "5": [ + -0.0005032679630955919, + 0.0037358976139699415, + 0.0005151417071530209, + 0.0023007474116432808, + -0.0019129948799514356, + 0.005698809764232035, + -0.0022764888376876065, + 0.0009897007385131736, + 0.003388581141935666, + -0.002979749491673576 + ], + "6": [ + 0.0007723202866611282, + -0.0021518052254251093, + 0.0049411074609891715, + 0.0005733332953130411, + 0.0021648158339006443, + 0.0017072632008870792, + 0.0009320819676294085, + 0.002184069998805277, + 0.0026147377830603386, + 0.0020509617170077804 + ], + "7": [ + -0.00666820409952941, + -0.0007737151235513751, + 0.001762745052351469, + -0.0012954590224478015, + -0.001181248883096067, + 0.005485998928589886, + 0.006003377300421452, + 0.0015132212731687523, + 0.0034779405450414335, + -0.00021508681661698003 + ], + "8": [ + 0.0020999707394967026, + -0.0018027105160354922, + 0.0015950815572812525, + 0.0024563446850568464, + 0.0005396518076248825, + 0.0005501522604777009, + 0.0004466089787335633, + 0.0008627565379604122, + 0.000942310781786404, + 0.0005023700229792793 + ], + "9": [ + -0.0033216971050804844, + -0.0021978657449274017, + -0.0011076306155161349, + -0.0008711212885562362, + -0.0016211905200990487, + -0.0037261940544540836, + -0.0004352245717163292, + -0.0009774371995563795, + 0.004273284592785864, + 0.0034181026002775257 + ], + "10": [ + -0.0034132237435796063, + -0.0007883630461136851, + -0.0015455419601315857, + -0.00031535870362214217, + 0.0016598080332539816, + 0.0030938174871390956, + 0.0269128690439945, + 0.002141810506454162, + 0.011428296535474985, + 0.0019638252651070715 + ], + "11": [ + -0.005687635000552865, + 0.0020935522154663278, + 0.0012675827581998468, + -4.072748475120097e-05, + -0.0029128283716669335, + -0.00014329547249651743, + -0.0027774027464459073, + -0.00343080788491288, + 0.001032940267778954, + -0.002719508632452876 + ], + "12": [ + -0.00040852760835703483, + 0.0025886181300433567, + -0.0013268199651689059, + -0.0012913964304526692, + -0.0009944940927905392, + -0.0013786218583102447, + 0.0006167208077267166, + -0.0005248463226509077, + 0.003172848491595041, + -0.0017341322424722523 + ], + "13": [ + -0.0008944415037596754, + 0.0043239312682777, + -0.004251992970449703, + -0.0012324888465232514, + 0.0012603464904262672, + 0.0009263028757810592, + 0.01693594112879913, + -0.002749684302312555, + 0.0018643002373842882, + 0.004179127665824139 + ], + "14": [ + -0.0009205368796238912, + 0.002322246081598114, + 0.0023968285281326996, + -5.900914872929617e-05, + 0.00017694519693176657, + 0.0016443155468975376, + -0.002382089026916433, + 0.0023217862268200876, + 0.001873372258004163, + -0.0008322198868792813 + ], + "15": [ + -0.005491341212648148, + 0.0001260737381230026, + 0.00012373215420164417, + 0.00018413698217936998, + -0.0001765726807276997, + 0.00449649251404697, + -0.003340276617533637, + 0.003356066586676202, + 0.002460967033553459, + 0.001670616112303249 + ], + "16": [ + -0.0009441898311461362, + -0.0032780018612706815, + 0.0016371434793894602, + 0.0019123636169086362, + -0.000664320563919264, + -0.0013766600750558281, + 0.0022653007135499796, + 0.0008809861228961873, + -0.002123397146287915, + 0.00390536563303264 + ], + "17": [ + 0.0032881459075407783, + -0.0005485774004693981, + -0.0010844965583566206, + 0.0006458505624261521, + 0.010250155871104187, + -8.495374440864962e-05, + 0.00042279332267452184, + 0.0007111195359946463, + 0.0006332270392672667, + -0.000874536152313821 + ], + "18": [ + -0.002129665477549086, + -0.002254425353433893, + 0.0012646617913867767, + 0.007475677095042762, + 0.0009983434268992304, + 0.002606271700954516, + -0.00014066939705203064, + 0.00017980999686650927, + -0.0017928127148996741, + -0.00840516211897414 + ], + "19": [ + 0.0009015116794864334, + 0.002720026082741073, + 0.002132539450886128, + 0.001279107089667394, + 0.000166514743217894, + -0.0012601472087283027, + 0.001516088962088211, + 0.0023072025588714673, + 0.000422574720473772, + 0.005120351581613695 + ] + } + } +} \ No newline at end of file diff --git a/data/mi1_patching/gpt2-medium.json b/data/mi1_patching/gpt2-medium.json new file mode 100644 index 0000000000000000000000000000000000000000..e953b97a5c3f5891ee785039dfbefcbb21fafcd6 --- /dev/null +++ b/data/mi1_patching/gpt2-medium.json @@ -0,0 +1,916 @@ +{ + "model": "gpt2-medium", + "N": 24, + "L_crit": 23, + "N_sem": 1, + "n_heads": 16, + "layers_patched": [ + 21, + 22, + 23 + ], + "n_prompts": 10, + "head_summary": { + "21": { + "per_head_R_patch": { + "0": 0.00037104863302689264, + "1": -0.0005008152199040227, + "2": -0.00030629704742134807, + "3": -0.0021426075431239815, + "4": 1.2010753962762161e-05, + "5": -0.001290136055073442, + "6": 0.011071324238311505, + "7": 0.004823819396196785, + "8": 5.033000700348321e-06, + "9": -0.0007986484968689012, + "10": 0.0014972904167570062, + "11": -0.001525040229884943, + "12": 0.018735345162557124, + "13": -0.0009398496119430147, + "14": 0.00428500019077836, + "15": 0.0008638338157996856 + }, + "top_head": 12, + "max_R_patch": 0.018735345162557124, + "n_critical_01": 0 + }, + "22": { + "per_head_R_patch": { + "0": -0.006665112713764534, + "1": -0.004002053623451124, + "2": 0.0002846260580299788, + "3": 0.004456527716733996, + "4": -0.0009234391963858649, + "5": 0.001780976773309963, + "6": -0.0019336451489104802, + "7": -0.002455026336267262, + "8": 0.0030269870653014458, + "9": 0.0008804860349571474, + "10": 0.00264768863096515, + "11": 0.005312262804176642, + "12": 0.0008858839955232621, + "13": -0.0029225602587100766, + "14": -0.004178754872181187, + "15": -0.0002663539591700245 + }, + "top_head": 11, + "max_R_patch": 0.005312262804176642, + "n_critical_01": 0 + }, + "23": { + "per_head_R_patch": { + "0": 0.0005743887627727713, + "1": 0.004271333296196305, + "2": -0.0015311662840133914, + "3": -0.01287815766505287, + "4": -0.002577586812838797, + "5": 0.009574232347929076, + "6": 0.0007297107343630603, + "7": 0.0009666670392251197, + "8": -0.002185699921739419, + "9": -0.0004018856643018012, + "10": 0.0004520175239430377, + "11": 0.004707340784695038, + "12": -0.0017123351524212203, + "13": 0.005016305247761666, + "14": 0.0021559479079406576, + "15": -0.012412254665971489 + }, + "top_head": 5, + "max_R_patch": 0.009574232347929076, + "n_critical_01": 0 + } + }, + "wov_static": { + "21": { + "0": { + "stable_rank_OV": 11.679811747870708, + "frob_norm_OV": 63.2559700012207, + "spec_norm_OV": 18.5090274810791 + }, + "1": { + "stable_rank_OV": 4.185392197540481, + "frob_norm_OV": 46.041358947753906, + "spec_norm_OV": 22.505054473876953 + }, + "2": { + "stable_rank_OV": 26.26257210898079, + "frob_norm_OV": 59.34210968017578, + "spec_norm_OV": 11.579621315002441 + }, + "3": { + "stable_rank_OV": 6.014958249298342, + "frob_norm_OV": 76.7730941772461, + "spec_norm_OV": 31.303489685058594 + }, + "4": { + "stable_rank_OV": 14.34388687544991, + "frob_norm_OV": 66.28335571289062, + "spec_norm_OV": 17.50132942199707 + }, + "5": { + "stable_rank_OV": 11.23461176677878, + "frob_norm_OV": 60.93190383911133, + "spec_norm_OV": 18.178821563720703 + }, + "6": { + "stable_rank_OV": 11.800190352603545, + "frob_norm_OV": 37.69043731689453, + "spec_norm_OV": 10.97202205657959 + }, + "7": { + "stable_rank_OV": 21.15500397806295, + "frob_norm_OV": 74.7559585571289, + "spec_norm_OV": 16.25321388244629 + }, + "8": { + "stable_rank_OV": 17.081108800627113, + "frob_norm_OV": 63.637603759765625, + "spec_norm_OV": 15.397697448730469 + }, + "9": { + "stable_rank_OV": 24.2770936400285, + "frob_norm_OV": 55.69814682006836, + "spec_norm_OV": 11.304266929626465 + }, + "10": { + "stable_rank_OV": 11.642961077890694, + "frob_norm_OV": 55.43775177001953, + "spec_norm_OV": 16.247026443481445 + }, + "11": { + "stable_rank_OV": 14.62375122981695, + "frob_norm_OV": 54.711456298828125, + "spec_norm_OV": 14.30700969696045 + }, + "12": { + "stable_rank_OV": 24.083121970599645, + "frob_norm_OV": 72.73763275146484, + "spec_norm_OV": 14.821861267089844 + }, + "13": { + "stable_rank_OV": 12.623982388705372, + "frob_norm_OV": 61.972679138183594, + "spec_norm_OV": 17.442232131958008 + }, + "14": { + "stable_rank_OV": 9.868740642181328, + "frob_norm_OV": 62.95900344848633, + "spec_norm_OV": 20.041349411010742 + }, + "15": { + "stable_rank_OV": 22.692937137362303, + "frob_norm_OV": 91.3383560180664, + "spec_norm_OV": 19.173784255981445 + } + }, + "22": { + "0": { + "stable_rank_OV": 12.311270878027692, + "frob_norm_OV": 94.58307647705078, + "spec_norm_OV": 26.95640754699707 + }, + "1": { + "stable_rank_OV": 23.34768785947122, + "frob_norm_OV": 83.52806091308594, + "spec_norm_OV": 17.28663444519043 + }, + "2": { + "stable_rank_OV": 2.8542139389081433, + "frob_norm_OV": 64.43232727050781, + "spec_norm_OV": 38.13823318481445 + }, + "3": { + "stable_rank_OV": 23.48007769312233, + "frob_norm_OV": 76.95944213867188, + "spec_norm_OV": 15.882253646850586 + }, + "4": { + "stable_rank_OV": 24.78080081566164, + "frob_norm_OV": 86.60465240478516, + "spec_norm_OV": 17.397367477416992 + }, + "5": { + "stable_rank_OV": 34.173097601051595, + "frob_norm_OV": 101.55699920654297, + "spec_norm_OV": 17.37271499633789 + }, + "6": { + "stable_rank_OV": 20.639135421474517, + "frob_norm_OV": 74.37921142578125, + "spec_norm_OV": 16.372154235839844 + }, + "7": { + "stable_rank_OV": 13.824386911378532, + "frob_norm_OV": 82.35218811035156, + "spec_norm_OV": 22.148900985717773 + }, + "8": { + "stable_rank_OV": 3.943176537153343, + "frob_norm_OV": 83.95404052734375, + "spec_norm_OV": 42.27839660644531 + }, + "9": { + "stable_rank_OV": 11.992220712900851, + "frob_norm_OV": 72.34245300292969, + "spec_norm_OV": 20.890239715576172 + }, + "10": { + "stable_rank_OV": 20.1773902634747, + "frob_norm_OV": 75.23760986328125, + "spec_norm_OV": 16.74952507019043 + }, + "11": { + "stable_rank_OV": 8.249568472060268, + "frob_norm_OV": 80.56853485107422, + "spec_norm_OV": 28.051097869873047 + }, + "12": { + "stable_rank_OV": 27.77376196172572, + "frob_norm_OV": 79.19612121582031, + "spec_norm_OV": 15.027493476867676 + }, + "13": { + "stable_rank_OV": 1.5862925974932272, + "frob_norm_OV": 94.48916625976562, + "spec_norm_OV": 75.02230072021484 + }, + "14": { + "stable_rank_OV": 8.846231292410483, + "frob_norm_OV": 77.90113067626953, + "spec_norm_OV": 26.191755294799805 + }, + "15": { + "stable_rank_OV": 18.983688030620996, + "frob_norm_OV": 77.83248901367188, + "spec_norm_OV": 17.863666534423828 + } + }, + "23": { + "0": { + "stable_rank_OV": 20.490594134920688, + "frob_norm_OV": 122.0879135131836, + "spec_norm_OV": 26.970897674560547 + }, + "1": { + "stable_rank_OV": 5.089024892344815, + "frob_norm_OV": 81.949951171875, + "spec_norm_OV": 36.32715606689453 + }, + "2": { + "stable_rank_OV": 2.974450009762857, + "frob_norm_OV": 97.28901672363281, + "spec_norm_OV": 56.41056823730469 + }, + "3": { + "stable_rank_OV": 9.493191838313548, + "frob_norm_OV": 97.42037963867188, + "spec_norm_OV": 31.618677139282227 + }, + "4": { + "stable_rank_OV": 1.401894216194655, + "frob_norm_OV": 146.4619598388672, + "spec_norm_OV": 123.69928741455078 + }, + "5": { + "stable_rank_OV": 1.5957646783241533, + "frob_norm_OV": 191.07620239257812, + "spec_norm_OV": 151.25933837890625 + }, + "6": { + "stable_rank_OV": 1.2050007794878974, + "frob_norm_OV": 170.42822265625, + "spec_norm_OV": 155.2558135986328 + }, + "7": { + "stable_rank_OV": 2.6494666521275536, + "frob_norm_OV": 97.75690460205078, + "spec_norm_OV": 60.05763244628906 + }, + "8": { + "stable_rank_OV": 3.4537874627687497, + "frob_norm_OV": 92.5389404296875, + "spec_norm_OV": 49.793968200683594 + }, + "9": { + "stable_rank_OV": 18.231310052865787, + "frob_norm_OV": 89.03492736816406, + "spec_norm_OV": 20.85218048095703 + }, + "10": { + "stable_rank_OV": 11.362583342119331, + "frob_norm_OV": 96.81660461425781, + "spec_norm_OV": 28.721776962280273 + }, + "11": { + "stable_rank_OV": 12.755016594400372, + "frob_norm_OV": 68.3976058959961, + "spec_norm_OV": 19.15139389038086 + }, + "12": { + "stable_rank_OV": 21.869158110566932, + "frob_norm_OV": 101.1629409790039, + "spec_norm_OV": 21.632434844970703 + }, + "13": { + "stable_rank_OV": 20.135880603621256, + "frob_norm_OV": 91.51103973388672, + "spec_norm_OV": 20.39333152770996 + }, + "14": { + "stable_rank_OV": 22.102452745728687, + "frob_norm_OV": 107.53323364257812, + "spec_norm_OV": 22.87296485900879 + }, + "15": { + "stable_rank_OV": 1.858224250284883, + "frob_norm_OV": 143.62203979492188, + "spec_norm_OV": 105.35909271240234 + } + } + }, + "raw_patching": { + "21": { + "0": [ + 0.0014260509085222456, + 0.003932076944348191, + 0.0053704037896023455, + -0.0010185657068080248, + -0.0029763779680796287, + -0.0034990048880992504, + -0.00517899614975348, + 0.0018149787772968452, + 0.005447721550866702, + -0.0016078009276270177 + ], + "1": [ + -0.007658894116244124, + -0.010343094975840423, + -0.007035859735069383, + 0.01251626622231441, + -0.0008171841594758611, + -0.011147258657420537, + 0.004559761806534176, + -0.0003760722212356225, + -0.0031566305334658194, + 0.018450814170862962 + ], + "2": [ + 7.843279996872352e-05, + -0.0006157654373048185, + 0.00584538764470306, + -0.002503597499877097, + -0.0009676341331761141, + -0.0031605594748158406, + 0.0012109304398264253, + 0.0006647203436262172, + -0.003595674082501433, + -1.9211074662603397e-05 + ], + "3": [ + 0.003555370080878883, + -0.009841837208681974, + -0.0017413205684536123, + 2.6141895516935996e-05, + -0.007644649245055194, + -0.0005040692659829605, + -0.0007727262095822908, + 0.00019181293075205111, + -0.003781085925351656, + -0.0009137119152799949 + ], + "4": [ + -0.0028943829755603266, + -0.0006088843908707533, + 5.373231806342526e-05, + 0.001853492666095845, + 0.0017006404003901375, + 0.002464832093573067, + -0.0006884561653045727, + 0.0004034386884378196, + -0.0007673516413361067, + -0.0013969534538609137 + ], + "5": [ + -0.00015448884842324327, + -0.008502326836183927, + 0.0005813418759814065, + -0.001688034907638859, + -0.002071002544950473, + 3.323197044457008e-05, + 0.002072591642566094, + -0.0013230015184989763, + -0.0006860969112565412, + -0.0011635744727744725 + ], + "6": [ + 0.014312922710719897, + 0.001679681078264147, + 0.04158217448961421, + -0.0021905402325755174, + 0.005667394939530802, + 0.010797940110950194, + 0.005906577692758422, + 0.0208986936798534, + -0.008075282679931392, + 0.020133680593930895 + ], + "7": [ + 0.009956212092999479, + 0.007208337232403542, + 0.0006669201216842532, + 0.0028639898866332104, + -0.002152504856284673, + -0.00559851502086346, + 0.03102341487195425, + -0.0018297145673287975, + 0.0079529742106662, + -0.001852920009896161 + ], + "8": [ + -0.0005266380986735661, + 0.004425571479632311, + 0.00011053856736617691, + -0.0006127746373023351, + -0.0006875213914441793, + -0.0004959527017729964, + -0.00014009894861170648, + -0.0009572072012352195, + 0.00015763661277804763, + -0.0012232236737330497 + ], + "9": [ + 0.000500618924202283, + -0.0009986339183797345, + -0.00499390869140965, + -0.0005922268922663895, + -0.0005531249033097536, + -0.0010813407525765409, + 0.0005057707478882336, + -0.00036368920440204915, + -0.0008618848805291245, + 0.00045193460209371324 + ], + "10": [ + -0.002702991932574446, + 0.004647000025138771, + 0.006084785409621798, + 9.897343158675357e-05, + 0.006409765739991558, + -1.5620557536157368e-05, + 0.00019788412183071325, + -0.002575419841046584, + 0.0033497171109412487, + -0.0005211893403835922 + ], + "11": [ + -0.0016139393703292993, + -0.0013957938035869398, + -0.0018055042527307472, + -0.0006061046886519235, + -0.00021013600977832874, + 0.0001024524803106792, + 0.002104794838057936, + -0.013928788824908306, + 0.0019546209057370745, + 0.00014799642703042617 + ], + "12": [ + 0.023520708261115113, + -0.0003509333681373307, + 0.009433157229995246, + 9.832795268510083e-05, + 0.12812327992870245, + 0.018809831271149966, + 0.007941548779843393, + 0.0008072488673806464, + 0.0006466268474697658, + -0.0016763441446330965 + ], + "13": [ + 0.0015525191075148764, + -0.0021581784610637655, + -0.004674711671517998, + 0.0011971482029319498, + 0.0024029803938950805, + 0.0005124155442743387, + -0.00781499323120489, + 0.0018371443774289414, + -0.001774203731452462, + -0.00047861665023621796 + ], + "14": [ + -0.012398136635566387, + 0.014571056916853422, + 0.020643415588069865, + 0.001827458350395851, + 0.013097277361663679, + 0.012729529250200712, + -0.0007245718985664518, + -0.0003267878142380006, + -0.0037787713228456415, + -0.0027904678881834604 + ], + "15": [ + -0.0024063983620388105, + 0.0032093024131392544, + 0.0009666898961433625, + -0.0015700198484533497, + 0.0021354223011312927, + -0.0015022535210876048, + 0.006011463801439796, + -0.0002796085201020861, + 0.0027161251302158813, + -0.0006423851323908803 + ] + }, + "22": { + "0": [ + 0.011265927269510699, + -0.016437761308453702, + -0.06247015204708488, + 0.0009854311231898508, + -0.017112706892295604, + -0.003804294902298561, + 0.013860165050355774, + 0.004522649238125993, + 0.0007756354818839484, + 0.0017639798494211453 + ], + "1": [ + -0.0008347401502253461, + -0.0009626407524169312, + 6.885605976091108e-05, + -0.006374857212539409, + -0.005450055442639603, + 0.003285600506465571, + -0.0007913860051009284, + -0.001065682428697322, + -0.027227156562987964, + -0.0006684742461302183 + ], + "2": [ + -0.016453875456277637, + -0.00017308478337995043, + 0.0028431404818777626, + 0.0073824497780193375, + -0.0007075882484014634, + -0.0015863288748851577, + -0.008071414937364232, + 0.0042829140322280124, + 0.01036576459140964, + 0.004964283997073478 + ], + "3": [ + -0.001748413469159248, + 0.007575855686817851, + 0.030386179172490063, + -0.0011121601475476723, + 0.0030167174959117075, + 0.007070905711367235, + 0.001692774514428664, + -0.0008653252163301053, + -3.9835527340356706e-05, + -0.0014114210532981829 + ], + "4": [ + -0.00024067736385936848, + -0.0035153325423671077, + -0.0002828016740180277, + -0.0016870666892863798, + 0.002658189651608232, + -0.003975508464312522, + 0.0001572539219110991, + -0.0002380015835412797, + -0.0007398200536329856, + -0.0013706271663603091 + ], + "5": [ + 0.0033295786870295276, + -0.005454023265892995, + 0.00036715034722514377, + 0.0010794558831972666, + 0.0031076872474513953, + 0.0016358705451101274, + 0.011328903595363813, + 0.00011726716941393958, + 0.0012692549215876957, + 0.0010286226026137153 + ], + "6": [ + -0.000980347453516565, + 0.0019466303924882704, + -0.009029734494146052, + 0.002416565427970919, + -0.004111544630109376, + 0.00029189002611682295, + -0.0028240998588570307, + -0.0037189914456270826, + -0.00037009275859328344, + -0.002956726694831423 + ], + "7": [ + 0.0025986650503193975, + -0.008257520376510449, + -0.0020521564953742966, + -0.0036817040752102907, + 0.001650936339311326, + -0.0040662455264710825, + 0.0059884400214853485, + -0.016480928594307772, + 0.0018980958761165073, + -0.002147845582031313 + ], + "8": [ + 0.00013735121908398473, + 0.014757286263293317, + 0.011809306077994543, + -0.0056593438500573455, + 0.010643563837102235, + 0.0019618348266367052, + 0.0027547275545498377, + -0.0018203034745352816, + -0.0033188963512558963, + -0.0009956554497976427 + ], + "9": [ + -0.0006556081413653588, + 0.003556971695147605, + -0.008122555906796048, + 0.0018420892054999798, + -0.0022631298369466236, + 0.005612833865271605, + 0.0037588954214520043, + 0.005855061849418485, + -0.0032244849332474057, + 0.0024447871311372325 + ], + "10": [ + -0.00023054489687776306, + 0.002105070897559825, + 0.0070978301888542025, + 4.518352311569185e-06, + 0.0029318192549385824, + 0.0028682100205361894, + 0.009376396765893469, + 0.0008368442776128867, + -0.0028892330334288688, + 0.004375974482251408 + ], + "11": [ + 0.0031620802513706425, + 0.0005305463237752406, + 0.011598803266725473, + 0.0012797695023435006, + 0.005844600722507434, + 0.008628673468796576, + 4.664948879659399e-06, + 0.0011599171868008152, + 0.011052836072142398, + 0.009860736298424677 + ], + "12": [ + -0.0009542031868603237, + 0.002937942171713807, + -0.007291610814181338, + 0.00013232317483881183, + 0.002061946732580006, + 0.006124943124104058, + 0.002707927583531319, + -0.0012950159004551005, + 0.0016359366870142208, + 0.0027986503829471616 + ], + "13": [ + -0.017994510806221494, + -0.005454111484437021, + -0.00810939948109173, + -0.0008284721702712926, + 0.0062685773925792825, + -0.0023230219339801087, + -0.003812015645791352, + 0.0031309219762006846, + -0.0010111158315748033, + 0.0009075453974870605 + ], + "14": [ + 0.0010503990770930963, + -0.005289583899827639, + -0.00228528344056481, + -0.0004877668900155877, + -0.008381360162784408, + -0.021379412985847855, + -0.005338657787215371, + 0.00023317220697618608, + -0.0002058778018507732, + 0.0002968229622252858 + ], + "15": [ + 0.0040811325342577425, + -0.002365756695158069, + -0.001709351683564618, + -0.001487506128858741, + -0.0006905056932480832, + -0.000534621238811033, + -0.0011263594311048581, + 0.0033914606503790677, + -0.002812120223623224, + 0.000590088318031571 + ] + }, + "23": { + "0": [ + -0.00454209723582831, + 0.001771516582595711, + 0.005941294299370042, + 0.0013718578256459583, + -0.0007048097605150702, + -0.0010833316079487962, + 0.0013915091061358215, + 0.002595108837811966, + 5.847416857300067e-06, + -0.0010030078363969107 + ], + "1": [ + 0.008675768487768451, + 0.002937677516081728, + -0.002479801809389384, + 0.0049804076253356045, + 0.011250406172882256, + 0.0013063074096413955, + 0.0012994139863180296, + 0.003997733154550819, + 0.006901901030566512, + 0.0038435193882076463 + ], + "2": [ + -0.009477234116794945, + -0.0008591604002738722, + -0.0009671817251416547, + -0.0008429954455584794, + 0.0025660879235222356, + 0.002194535191109167, + 0.0027804600144989266, + -0.008620189508355412, + 0.0016555498977230815, + -0.003741534670862962 + ], + "3": [ + -0.006158913726411646, + -0.022518136236934695, + -0.012885304969008353, + -0.0005601681068176368, + -0.036638067432609285, + -0.05512387554605275, + 0.0012231194998023097, + -0.0017789441983111468, + 0.007186231675252728, + -0.0015275176094382368 + ], + "4": [ + -0.02620643747216534, + 0.004511231685882021, + -0.0016326263598310312, + -0.0014600732755384995, + 0.0008214033447848286, + 0.011804853501148573, + -0.0009602270581002138, + -0.0021781726610255515, + -0.006002129761649465, + -0.0044736900718932915 + ], + "5": [ + 0.0462758523502867, + -0.0011661609334860166, + 0.0018281283866521895, + 0.0070008641673256254, + 0.011809705493717328, + 0.0014468158561063405, + -0.0060135705525467395, + 0.020839379029220582, + 0.005778953351595845, + 0.007942356330418905 + ], + "6": [ + 0.0005530325496997235, + -0.0009819606135587301, + 0.0007216360977442629, + -0.001999048158418538, + 0.0074746469491914335, + 0.002862850025303194, + 0.0025198248061256982, + -0.000480832543647653, + -0.005291303149933925, + 0.0019182613811251394 + ], + "7": [ + -0.0018537410888939613, + 0.001780955966806544, + 0.002618620544157363, + -0.000796520964639482, + -0.004182241710774269, + 0.0002480912079272052, + 0.018102710333073112, + -0.004159703014733958, + 0.0008805235217617685, + -0.0029720244024331256 + ], + "8": [ + -0.007489894574365742, + 0.0018300936958292925, + 0.012745994405242081, + -0.0022363692345928623, + -0.0043089201770020475, + -0.014976286109151346, + -0.007187783330223588, + 0.01116737607102145, + -0.008934365673216394, + -0.002466844290935036 + ], + "9": [ + -0.0015523940153299182, + 0.0005226066548128575, + -0.004459044655766858, + -0.0005517768810961511, + -0.0008210946239085627, + 0.0017214007547566362, + 0.003875368661221565, + -0.0006416879323157708, + -0.0006954771424651267, + -0.001416757462926684 + ], + "10": [ + 0.006610496514110357, + 0.0015875809183005037, + 0.0021254390161198375, + 0.002611930375537815, + -0.004194590545824906, + -0.0032327662677403327, + -0.0008938643982315108, + 0.0015997619447293404, + -0.0014708689819800211, + -0.0002229433355907061 + ], + "11": [ + -0.010578420620981027, + 0.0037469944389806396, + 0.004423632967889818, + 0.002122872527718922, + 0.006180489035884743, + 0.008725306525711431, + 0.007256854670086931, + 0.004912590438215217, + -0.00035778881895604783, + 0.02064087668239975 + ], + "12": [ + 0.002469820099812563, + -0.010122548615774227, + 0.0026848945016772403, + -0.00035802563078338683, + -0.0006263946579435294, + -0.0016793630778578596, + -0.0034242229598919235, + -0.0015969138508576187, + -0.0047521225872170485, + 0.00028152525462358315 + ], + "13": [ + 0.001119700147559879, + 0.008554287558615523, + -0.0010705887720325945, + 0.00022430391832432739, + 0.007610587041707189, + 0.012553644835197999, + 0.007047834863833806, + 0.0016405020701117969, + 0.009231365721093426, + 0.003251415093205308 + ], + "14": [ + -0.0012905760722126322, + 0.008588692790785848, + 0.0036444528773453655, + -2.9907189109910318e-05, + 0.00021188542807716888, + 0.0038337748760800343, + -0.001788180243128795, + -0.0023894269282063126, + 0.010002493819149872, + 0.0007762697206259373 + ], + "15": [ + -0.08072836665488946, + -0.0014918637980317747, + -0.007041515768549744, + -0.0033323924095992156, + -0.0063575919119026195, + -0.0005839331949545886, + -0.0089336780689916, + -0.010254128579545416, + -0.001073975562790779, + -0.004325100710459698 + ] + } + } +} \ No newline at end of file diff --git a/data/mi1_patching/gpt2-medium_h3_heatmap.json b/data/mi1_patching/gpt2-medium_h3_heatmap.json new file mode 100644 index 0000000000000000000000000000000000000000..cabfada653948ee78157b145697e1f59ffb15551 --- /dev/null +++ b/data/mi1_patching/gpt2-medium_h3_heatmap.json @@ -0,0 +1,497 @@ +{ + "model": "gpt2-medium", + "L_crit": 23, + "N": 24, + "H": 16, + "n_prompts": 10, + "mu_baseline": -9.4803, + "mu_d10": -4.7681, + "denom": 4.7122, + "R_matrix": [ + [ + -0.0006, + -0.0004, + 0.0039, + 0.0022, + -0.0011, + -0.0039, + -0.0001, + -0.0062, + 0.0011, + -0.0022, + -0.001, + -0.0007, + 0.0054, + -0.0018, + 0.0026, + -0.009 + ], + [ + -0.0115, + 0.0481, + -0.0078, + -0.0053, + -0.0208, + -0.003, + 0.0037, + 0.007, + 0.0221, + 0.0059, + -0.0162, + 0.0071, + -0.0035, + -0.0213, + 0.0089, + 0.0127 + ], + [ + 0.0175, + 0.0026, + -0.0076, + 0.0128, + 0.0065, + 0.0023, + -0.0046, + -0.007, + 0.0122, + 0.0031, + 0.0018, + 0.0126, + -0.011, + -0.0187, + -0.1426, + -0.0012 + ], + [ + -0.0075, + 0.0078, + 0.0027, + 0.0143, + 0.062, + -0.0046, + -0.0473, + -0.0136, + -0.0017, + -0.0027, + 0.0005, + 0.0057, + 0.0021, + 0.0012, + 0.006, + 0.0208 + ], + [ + 0.0001, + -0.0034, + 0.0135, + 0.0013, + 0.0052, + -0.001, + 0.0066, + 0.0013, + 0.0006, + 0.0039, + -0.0006, + 0.0044, + 0.0007, + 0.0174, + 0.0008, + 0.0035 + ], + [ + -0.0022, + 0.0048, + 0.0012, + 0.0008, + 0.0251, + -0.002, + 0.0011, + 0.0015, + -0.0083, + 0.0063, + -0.0004, + 0.0131, + -0.0086, + 0.0099, + -0.0114, + -0.0014 + ], + [ + -0.0356, + 0.0404, + 0.0057, + 0.0043, + 0.0149, + -0.0182, + -0.0013, + 0.0276, + -0.0019, + -0.0126, + 0.0049, + 0.0019, + 0.0078, + 0.0004, + 0.014, + -0.0208 + ], + [ + -0.045, + -0.0146, + 0.0334, + 0.0051, + -0.0277, + -0.009, + 0.0024, + -0.0025, + -0.0058, + -0.0152, + -0.0011, + 0.0546, + -0.0042, + 0.0081, + -0.0013, + 0.0019 + ], + [ + 0.0017, + 0.0031, + -0.0014, + 0.0159, + -0.0242, + 0.0012, + -0.0004, + -0.0008, + -0.0528, + 0.0075, + -0.005, + -0.0098, + -0.0331, + 0.0019, + -0.0221, + -0.008 + ], + [ + -0.0029, + 0.0045, + -0.0005, + 0.0134, + -0.0424, + -0.0004, + 0.0033, + 0.024, + -0.0003, + 0.0189, + 0.0008, + 0.0052, + -0.0234, + -0.097, + 0.0184, + -0.0027 + ], + [ + -0.0123, + 0.112, + 0.0097, + -0.0089, + 0.0075, + -0.0007, + 0.0404, + -0.0253, + 0.0076, + 0.0025, + -0.0013, + 0.0043, + -0.0127, + 0.0171, + -0.0477, + 0.0018 + ], + [ + -0.0034, + 0.0154, + -0.0004, + -0.001, + 0.0514, + 0.0029, + -0.0012, + 0.0303, + -0.0081, + -0.0015, + -0.0191, + -0.0088, + 0.0011, + -0.0002, + 0.0101, + 0.0066 + ], + [ + 0.0225, + 0.0489, + -0.001, + -0.0232, + 0.0213, + -0.0703, + 0.005, + 0.0122, + -0.0004, + -0.0003, + 0.0019, + -0.0077, + 0.0146, + 0.04, + 0.0018, + -0.0046 + ], + [ + 0.0034, + 0.0005, + -0.0, + -0.0014, + -0.001, + 0.0024, + -0.0007, + 0.012, + 0.0002, + -0.0075, + -0.0003, + 0.0013, + -0.0007, + 0.0007, + 0.082, + -0.011 + ], + [ + -0.0008, + 0.0001, + -0.0005, + 0.0012, + -0.0005, + 0.0053, + -0.0137, + -0.0005, + 0.0009, + 0.0432, + -0.0042, + 0.0007, + 0.0011, + 0.0021, + 0.007, + -0.0012 + ], + [ + 0.0014, + -0.0086, + -0.0021, + 0.0064, + 0.0066, + 0.0065, + -0.0014, + -0.002, + -0.002, + 0.0028, + 0.0006, + -0.0226, + 0.048, + 0.0191, + 0.0178, + -0.0009 + ], + [ + 0.0003, + -0.0, + -0.0006, + -0.0, + -0.001, + -0.0004, + -0.0028, + 0.003, + -0.0011, + -0.0003, + 0.0, + 0.0161, + 0.0276, + 0.0076, + 0.0046, + 0.0375 + ], + [ + 0.0004, + 0.0037, + 0.0206, + -0.0052, + -0.0017, + -0.0006, + -0.0031, + 0.009, + 0.0058, + 0.0017, + 0.0012, + 0.0021, + -0.0371, + 0.0001, + 0.0065, + -0.0013 + ], + [ + 0.004, + 0.001, + -0.0, + 0.002, + 0.0032, + 0.0388, + -0.0005, + 0.0042, + 0.0492, + -0.0296, + 0.0002, + 0.0042, + -0.0065, + 0.0177, + 0.0007, + 0.0056 + ], + [ + -0.0022, + -0.0032, + 0.0056, + -0.0006, + 0.0094, + 0.0009, + 0.0043, + -0.0059, + 0.0051, + 0.0058, + 0.0215, + 0.0015, + 0.0066, + 0.0, + 0.0023, + 0.0177 + ], + [ + 0.0021, + 0.0083, + -0.0014, + 0.006, + -0.0009, + 0.0019, + 0.0011, + 0.0173, + 0.0168, + -0.004, + 0.0092, + 0.0118, + 0.0302, + 0.0098, + -0.0014, + 0.0028 + ], + [ + 0.0048, + 0.0098, + -0.0001, + -0.0, + -0.0017, + 0.0002, + 0.005, + 0.0452, + -0.0002, + 0.0028, + 0.008, + -0.0001, + 0.0008, + 0.0171, + 0.0039, + 0.0172 + ], + [ + -0.0021, + 0.0142, + 0.0013, + 0.0098, + 0.0183, + -0.0001, + 0.0049, + 0.0052, + 0.0021, + 0.0017, + 0.0185, + 0.0038, + 0.0025, + -0.0006, + -0.0476, + 0.0217 + ], + [ + 0.0089, + -0.0003, + 0.0026, + -0.0153, + 0.008, + -0.0021, + 0.0043, + 0.0016, + 0.0042, + 0.0001, + 0.0033, + 0.0059, + 0.0014, + 0.0208, + 0.0167, + 0.0011 + ] + ], + "top10_heads": [ + { + "R": 0.112, + "l": 10, + "h": 1 + }, + { + "R": 0.082, + "l": 13, + "h": 14 + }, + { + "R": 0.062, + "l": 3, + "h": 4 + }, + { + "R": 0.0546, + "l": 7, + "h": 11 + }, + { + "R": 0.0514, + "l": 11, + "h": 4 + }, + { + "R": 0.0492, + "l": 18, + "h": 8 + }, + { + "R": 0.0489, + "l": 12, + "h": 1 + }, + { + "R": 0.0481, + "l": 1, + "h": 1 + }, + { + "R": 0.048, + "l": 15, + "h": 12 + }, + { + "R": 0.0452, + "l": 21, + "h": 7 + } + ], + "lcrit_n_crit_01": 0 +} \ No newline at end of file diff --git a/data/mi1_patching/gpt2.json b/data/mi1_patching/gpt2.json new file mode 100644 index 0000000000000000000000000000000000000000..623a08b8ab7469b859e951f1b6ff38fca4aae093 --- /dev/null +++ b/data/mi1_patching/gpt2.json @@ -0,0 +1,700 @@ +{ + "model": "gpt2", + "N": 12, + "L_crit": 11, + "N_sem": 1, + "n_heads": 12, + "layers_patched": [ + 9, + 10, + 11 + ], + "n_prompts": 10, + "head_summary": { + "9": { + "per_head_R_patch": { + "0": 0.0034528531721228994, + "1": 0.006254909422070402, + "2": 0.01243104662442615, + "3": 0.027916363802354245, + "4": 0.004531973112955019, + "5": -0.016982836120292513, + "6": 0.00449462152398525, + "7": 0.0031615059098070762, + "8": 0.002905575965479199, + "9": 0.0008253765000630503, + "10": 0.0060280842562661615, + "11": 0.01484479301529274 + }, + "top_head": 3, + "max_R_patch": 0.027916363802354245, + "n_critical_01": 0 + }, + "10": { + "per_head_R_patch": { + "0": 0.013516314524068426, + "1": -0.001099695932518027, + "2": 0.015096160928058244, + "3": 0.0043824130639881385, + "4": 0.0005623892327892135, + "5": 0.0029453003209625574, + "6": 0.00047144957750291683, + "7": -0.012929111752231043, + "8": 0.041266236837081166, + "9": 0.005351100386674195, + "10": 0.0022642277358425394, + "11": 0.012216394611916679 + }, + "top_head": 8, + "max_R_patch": 0.041266236837081166, + "n_critical_01": 0 + }, + "11": { + "per_head_R_patch": { + "0": 0.006252604810670329, + "1": -0.0015980218753392852, + "2": 0.001178217796582857, + "3": 0.0069905712884270445, + "4": 0.008224604804011573, + "5": 0.0041407533089216336, + "6": -0.0006012350414440811, + "7": 0.0015098899083710756, + "8": 0.010296707275400252, + "9": -0.005729302658649851, + "10": -0.002374579453295419, + "11": 0.01190372013724451 + }, + "top_head": 11, + "max_R_patch": 0.01190372013724451, + "n_critical_01": 0 + } + }, + "wov_static": { + "9": { + "0": { + "stable_rank_OV": 27.408270051027085, + "frob_norm_OV": 31.605144500732422, + "spec_norm_OV": 6.0369415283203125 + }, + "1": { + "stable_rank_OV": 19.919481210910376, + "frob_norm_OV": 48.7977409362793, + "spec_norm_OV": 10.933537483215332 + }, + "2": { + "stable_rank_OV": 33.963115826977834, + "frob_norm_OV": 34.24818801879883, + "spec_norm_OV": 5.876704216003418 + }, + "3": { + "stable_rank_OV": 10.186449376848554, + "frob_norm_OV": 22.90936279296875, + "spec_norm_OV": 7.177968978881836 + }, + "4": { + "stable_rank_OV": 19.74249129875483, + "frob_norm_OV": 47.84954071044922, + "spec_norm_OV": 10.769035339355469 + }, + "5": { + "stable_rank_OV": 15.582651852687384, + "frob_norm_OV": 53.30614471435547, + "spec_norm_OV": 13.50381851196289 + }, + "6": { + "stable_rank_OV": 35.83806450218393, + "frob_norm_OV": 51.29468536376953, + "spec_norm_OV": 8.56840705871582 + }, + "7": { + "stable_rank_OV": 19.411321133223833, + "frob_norm_OV": 40.665096282958984, + "spec_norm_OV": 9.229841232299805 + }, + "8": { + "stable_rank_OV": 17.724075819383255, + "frob_norm_OV": 39.05353927612305, + "spec_norm_OV": 9.276381492614746 + }, + "9": { + "stable_rank_OV": 37.93912156776617, + "frob_norm_OV": 51.800724029541016, + "spec_norm_OV": 8.409926414489746 + }, + "10": { + "stable_rank_OV": 24.766365082827907, + "frob_norm_OV": 31.11797523498535, + "spec_norm_OV": 6.2528815269470215 + }, + "11": { + "stable_rank_OV": 20.787730739716768, + "frob_norm_OV": 40.733482360839844, + "spec_norm_OV": 8.934041976928711 + } + }, + "10": { + "0": { + "stable_rank_OV": 38.22906285048873, + "frob_norm_OV": 59.05110549926758, + "spec_norm_OV": 9.550612449645996 + }, + "1": { + "stable_rank_OV": 35.337684185342425, + "frob_norm_OV": 50.376163482666016, + "spec_norm_OV": 8.474343299865723 + }, + "2": { + "stable_rank_OV": 30.911335920222314, + "frob_norm_OV": 55.27272415161133, + "spec_norm_OV": 9.94150161743164 + }, + "3": { + "stable_rank_OV": 19.051415661602288, + "frob_norm_OV": 41.480369567871094, + "spec_norm_OV": 9.503399848937988 + }, + "4": { + "stable_rank_OV": 4.6593835707170745, + "frob_norm_OV": 41.364315032958984, + "spec_norm_OV": 19.16291618347168 + }, + "5": { + "stable_rank_OV": 29.03794266776303, + "frob_norm_OV": 56.42047882080078, + "spec_norm_OV": 10.470172882080078 + }, + "6": { + "stable_rank_OV": 35.91256474485841, + "frob_norm_OV": 56.183597564697266, + "spec_norm_OV": 9.375325202941895 + }, + "7": { + "stable_rank_OV": 17.968746540541265, + "frob_norm_OV": 46.63233184814453, + "spec_norm_OV": 11.000900268554688 + }, + "8": { + "stable_rank_OV": 10.487604850112564, + "frob_norm_OV": 50.32241439819336, + "spec_norm_OV": 15.539009094238281 + }, + "9": { + "stable_rank_OV": 12.970670360230072, + "frob_norm_OV": 32.936763763427734, + "spec_norm_OV": 9.145337104797363 + }, + "10": { + "stable_rank_OV": 3.4739969204351477, + "frob_norm_OV": 74.28404998779297, + "spec_norm_OV": 39.854820251464844 + }, + "11": { + "stable_rank_OV": 16.658643653607324, + "frob_norm_OV": 30.82961654663086, + "spec_norm_OV": 7.553501129150391 + } + }, + "11": { + "0": { + "stable_rank_OV": 1.9895708090537703, + "frob_norm_OV": 86.97119903564453, + "spec_norm_OV": 61.658897399902344 + }, + "1": { + "stable_rank_OV": 17.881607190408033, + "frob_norm_OV": 77.23712158203125, + "spec_norm_OV": 18.2651309967041 + }, + "2": { + "stable_rank_OV": 31.527352839346545, + "frob_norm_OV": 106.23267364501953, + "spec_norm_OV": 18.91970443725586 + }, + "3": { + "stable_rank_OV": 25.760754015986006, + "frob_norm_OV": 68.92652893066406, + "spec_norm_OV": 13.580229759216309 + }, + "4": { + "stable_rank_OV": 2.8670004694928535, + "frob_norm_OV": 81.350830078125, + "spec_norm_OV": 48.04499053955078 + }, + "5": { + "stable_rank_OV": 17.95727829022412, + "frob_norm_OV": 80.75818634033203, + "spec_norm_OV": 19.05751609802246 + }, + "6": { + "stable_rank_OV": 27.181774531023397, + "frob_norm_OV": 97.40624237060547, + "spec_norm_OV": 18.683053970336914 + }, + "7": { + "stable_rank_OV": 22.42807228285055, + "frob_norm_OV": 76.02623748779297, + "spec_norm_OV": 16.053417205810547 + }, + "8": { + "stable_rank_OV": 1.022237249959398, + "frob_norm_OV": 338.5617980957031, + "spec_norm_OV": 334.8591003417969 + }, + "9": { + "stable_rank_OV": 23.55471692205203, + "frob_norm_OV": 90.40254211425781, + "spec_norm_OV": 18.6269474029541 + }, + "10": { + "stable_rank_OV": 19.22793877092154, + "frob_norm_OV": 62.81685256958008, + "spec_norm_OV": 14.32550048828125 + }, + "11": { + "stable_rank_OV": 2.3257237279019654, + "frob_norm_OV": 95.02363586425781, + "spec_norm_OV": 62.30925750732422 + } + } + }, + "raw_patching": { + "9": { + "0": [ + -0.006097191789151019, + 0.008568415684951878, + 0.031010387122832123, + -0.003027732244642194, + -0.0017992963546058299, + 0.054683656271904045, + -0.04873021441928796, + -0.018330081986559447, + 0.024015403988646912, + -0.005764814552859518 + ], + "1": [ + 0.001171407174980582, + -0.003329339003233901, + 0.007053922426411087, + 0.041583417780813835, + -0.0017505544624029487, + -0.00522034782589114, + 0.011528506627551253, + 0.010846391835210767, + 0.00764119919536436, + -0.006975509528099863 + ], + "2": [ + 0.007780646485138581, + 0.002546568179306427, + 0.022554507041561922, + 0.0012865469792261608, + 0.00043520152099059314, + 0.008577274990150922, + -0.00030652433011342056, + 0.03210807130880336, + 0.04756748596204258, + 0.0017606881071543805 + ], + "3": [ + -0.01139581452136456, + 0.009350583449692966, + 0.08196438306049411, + 0.0030271388189137684, + 0.031403934919521245, + 0.04244738677343549, + 0.02205902066504741, + 0.017523169970280236, + 0.047110669248097516, + 0.035673165639424256 + ], + "4": [ + 0.0006587969643900785, + -0.006238119606362701, + -0.006223194778363465, + 0.02865704767317257, + 0.0050854323426699774, + 0.0029663258627154548, + 0.03931105744582062, + -0.0056732897209239635, + -0.008676445617117242, + -0.004547879436451134 + ], + "5": [ + -0.00025771182925406216, + -0.0055753575605258015, + -0.03790481207545868, + 0.0013755608384899635, + -0.0154446526531835, + -0.024176432354967456, + -0.01552542980459573, + -0.01592980411487532, + -0.04391783122646168, + -0.012471890422092886 + ], + "6": [ + 0.0003897188055488285, + 0.0032581026368421963, + 0.012001137414498314, + 0.0004341650985591974, + 0.004657097070841877, + 0.007593934460365404, + 0.013729208236390695, + 0.0038861641732028736, + -0.0007204621196594074, + -0.0002828505367374804 + ], + "7": [ + -0.002842032548463793, + -0.006888368174082114, + -0.002123886238039706, + 0.012969096760087886, + -3.2381570124348766e-05, + 0.0052762732366598295, + 0.017051998012377827, + 0.002144847152586373, + 1.797992999902459e-05, + 0.006041532537069788 + ], + "8": [ + -0.02703723448833556, + 0.003124298879863068, + 0.008709092795781019, + -0.0017443007504902661, + -0.016061597855709705, + 0.04464776241781739, + 0.005809378978172935, + 0.00020072733236120742, + 0.014050547307277958, + -0.002642914961946065 + ], + "9": [ + 0.0035962616662195677, + 0.007654705635180603, + -0.002387573830426039, + 0.0034934972632400412, + 0.0016645144266013415, + -0.0037654982294523577, + 0.0009735036624248493, + 0.002482258827958683, + -0.005830376295663602, + 0.0003724718745474161 + ], + "10": [ + 0.015630391251133927, + 0.0009760512931638037, + 0.008254483095983606, + 0.003557958132990245, + 0.012842513757615087, + -0.0007640217558932595, + 0.00552624294972885, + 0.004913520460694135, + -0.00011817964039559881, + 0.009461883017640818 + ], + "11": [ + -0.04233181259602227, + -0.006651742825824377, + 0.016308127866833978, + 0.008076153272788761, + 0.007930772088203723, + -0.011890548777353014, + 0.07954884195072742, + 0.03523431797411587, + -0.01728838032961487, + 0.07951220152907218 + ] + }, + "10": { + "0": [ + 0.0001417977750480866, + -0.005528017414394595, + 0.021407857076788724, + 0.0030455350164949545, + -0.00478052001875039, + 0.03563675869336635, + -0.014188169262090143, + -0.0038413933797220136, + 0.1043972560643867, + -0.0011279593104434173 + ], + "1": [ + -0.001031972696183297, + -0.003915738679594824, + 0.020943331159269156, + 0.0036517193980814505, + 8.95155446369432e-05, + -0.005226834472281239, + -0.0036898485339506015, + -0.0054552174122804355, + -0.009425458781398216, + -0.0069364548514792065 + ], + "2": [ + -0.006503453668455458, + 0.01779032138074999, + 0.0019120143748775943, + 0.03762712262761807, + -0.00175046969389477, + -0.008541861407156329, + 0.02932837998036739, + 0.05023231141871322, + 0.016687814527084635, + 0.01417942974067811 + ], + "3": [ + 0.0023805145520572825, + 0.001993939817383053, + 0.023671827348283672, + 0.008603337854278633, + -0.006514205548025733, + 0.004330625381843803, + -0.009468685141708285, + 7.293784898206355e-05, + 0.009791381979870829, + 0.008962456546916062 + ], + "4": [ + -0.011658027867286815, + 0.016026222495777842, + 0.0018067069359690693, + 0.0006923052904242251, + -0.011692289870135273, + 0.0467629350852915, + -0.01679995466732945, + 0.00552311484080439, + -0.0070698168973049045, + -0.017967303018318455 + ], + "5": [ + 0.016583249791873728, + -0.0032336033573953137, + -0.005392467130315845, + -0.007552825958533654, + 0.0020507197498645167, + 0.002001393383496432, + 0.011599497002029944, + 0.005863787964708418, + 0.002856911490297274, + 0.00467634027360007 + ], + "6": [ + -0.0012642509562620673, + -0.008919697661021025, + -0.004579337277099091, + -0.00022438910356083585, + 0.0026769047197822246, + 0.009177552753386012, + 0.0010076230672130576, + -0.0015794305753148476, + 0.00465725362678252, + 0.003762267181123222 + ], + "7": [ + 0.0016840173855710856, + -0.0006505500973126063, + -0.038666964189681255, + -0.010392961494777385, + -0.0033469997769366147, + -0.07438517919153982, + 0.006887717263374456, + -0.0010660489268902824, + -0.021370185845021573, + 0.012016037350903574 + ], + "8": [ + 0.0040000477113565, + 0.01630875572459912, + 0.008740377764369228, + 0.09835816331823638, + 0.08014675957599221, + -0.0052341100351241876, + 0.10791830156265202, + 0.003347580422487758, + 0.0026226302918677725, + 0.09645386203437488 + ], + "9": [ + -0.0011420347787206213, + 0.007756622637679634, + 0.015953657999526503, + 0.003304936238032886, + 0.014599595395147812, + -0.0020806356583160175, + 0.010405868147423105, + 0.012004947302273423, + 0.00168631866081304, + -0.008978272077117815 + ], + "10": [ + -0.0028161488276216818, + 0.039555179711767315, + 0.0027217922671741823, + -0.007711567340887436, + -0.00232604786442966, + 0.010247674092985016, + -0.012509714671702471, + -0.0025305875652923268, + 0.00020166434049157227, + -0.0021899667840591147 + ], + "11": [ + 0.0038943746075706643, + 0.013257050094297125, + 0.022633557453262485, + 0.01812678230048077, + 0.021616308659658204, + 0.01735791511036981, + 0.006432333272047965, + 0.006599392856271424, + -0.0035672542523692906, + 0.015813486017577635 + ] + }, + "11": { + "0": [ + -0.05314400548135558, + 0.009601380688830747, + 0.011169292446143972, + 0.005264427993293395, + 0.015408032657650205, + -0.021504459986548645, + 0.12923935252090932, + 0.03244325926926768, + -0.02839817004087146, + -0.03755306196061634 + ], + "1": [ + 0.0014876387209211562, + -0.010723749217091164, + -0.007210067939275451, + 0.018628523753864406, + -0.005502239097385744, + -0.014074181861457763, + 0.006716019613472504, + 0.0031965160564049883, + -0.003482143226946772, + -0.00501653555589901 + ], + "2": [ + 0.0035906347703843263, + 0.00448774031788081, + 0.015231031091154212, + 0.00020858914354151086, + -0.0012223618879400767, + 0.0014995548621002744, + -0.01030681181094121, + 0.0011038520803261082, + -0.0029564787408446363, + 0.000146428140167254 + ], + "3": [ + -0.011150594400864734, + 0.0015912470456746009, + 0.05512020403135072, + 0.001517167052935463, + -0.006721464550523201, + 0.019511744684033563, + -0.014146345475575564, + 0.0009717634147915174, + 0.03100345286258942, + -0.0077914617801413425 + ], + "4": [ + 0.014412393378637546, + 0.022605145924846255, + 0.0008162024840602382, + 0.01624265561273028, + 0.017240219193429458, + -0.011670002800089655, + 0.0015527080743537095, + 0.0023254128031151397, + -0.006068723307409466, + 0.0247900366764422 + ], + "5": [ + 6.819797752312737e-05, + 0.002246244704486795, + 0.007678644142906885, + 0.002277419589264391, + 0.004642432118926923, + 0.023587813023757814, + 0.002004239874817016, + -0.005658613202531231, + 0.0016064480169982773, + 0.002954706843066346 + ], + "6": [ + 0.0012077569220762423, + 0.013918530639362955, + 0.012019154204444202, + -0.0029655708995896383, + -0.006891849251962837, + -0.005456584474827847, + -0.006532765390980998, + -0.009349386959152073, + 0.000512473180675716, + -0.0024741083844865342 + ], + "7": [ + -0.0006827675406482074, + -0.001208982903904934, + 0.0003888274667391689, + -0.005855034949508725, + 0.021206537691121286, + -0.012723995181097061, + 0.006330250214173568, + 0.006966157568429199, + -0.0034552185076517504, + 0.004133125226058214 + ], + "8": [ + -0.0518795294492601, + -0.004969509225403969, + 0.007284509404710787, + 0.035704942515032305, + -0.014225257663029687, + 0.014060419652224715, + 0.05603616954780819, + 0.016476244991598665, + -0.002235474512743047, + 0.04671455749306467 + ], + "9": [ + 0.0011400090962199344, + -0.017447557615688523, + -0.015211338320748242, + -0.00017439298594100003, + -0.007824048536406458, + 0.005171873292732699, + -0.008791249862768212, + -0.0016958049888329775, + 0.000626135250719801, + -0.013086651915785532 + ], + "10": [ + 0.01913617243232281, + 0.02697197287585691, + -0.039104674419839495, + 0.0015495087551346445, + 0.000916093267889626, + -0.026780733223208477, + -0.000509589819901306, + -0.006000768803040586, + -0.0023784104386649412, + 0.00245463484049662 + ], + "11": [ + 0.0017770862426859807, + 0.02028103120291752, + 0.03704000615805605, + 0.001335356245389146, + 0.011466975175395695, + 0.008257851484130382, + 0.010481261025745437, + 0.011927710271135913, + 0.0035571348947819, + 0.012912788672207073 + ] + } + } +} \ No newline at end of file diff --git a/data/mi1_patching/pythia-70m_h3_heatmap.json b/data/mi1_patching/pythia-70m_h3_heatmap.json new file mode 100644 index 0000000000000000000000000000000000000000..c395b3b7688d78fe60984d53ff0957a1defc72c3 --- /dev/null +++ b/data/mi1_patching/pythia-70m_h3_heatmap.json @@ -0,0 +1,125 @@ +{ + "model": "pythia-70m", + "L_crit": 4, + "N": 6, + "H": 8, + "n_prompts": 10, + "mu_baseline": -13.6788, + "mu_d10": -12.9395, + "denom": 0.7394, + "R_matrix": [ + [ + -0.6308, + -0.0023, + -0.1547, + 0.1323, + -1.1188, + -0.8701, + -0.3949, + -1.1709 + ], + [ + -0.3975, + -0.0665, + -0.0664, + 0.121, + -0.0383, + -0.1211, + -0.2912, + -0.1783 + ], + [ + -0.3745, + -0.4156, + -0.0985, + -0.3412, + -0.2426, + -0.529, + -0.1144, + -0.2443 + ], + [ + -0.13, + 0.1946, + -0.2168, + 2.8466, + -0.0267, + -0.1579, + -0.0434, + -0.7216 + ], + [ + -0.2634, + 0.093, + 0.0799, + -0.0939, + -0.2192, + -0.0841, + -0.0398, + -0.0249 + ], + [ + -0.0681, + 0.0806, + -0.1675, + 0.153, + 0.0786, + 0.0288, + 0.0763, + 0.1382 + ] + ], + "top10_heads": [ + { + "R": 2.8466, + "l": 3, + "h": 3 + }, + { + "R": 0.1946, + "l": 3, + "h": 1 + }, + { + "R": 0.153, + "l": 5, + "h": 3 + }, + { + "R": 0.1382, + "l": 5, + "h": 7 + }, + { + "R": 0.1323, + "l": 0, + "h": 3 + }, + { + "R": 0.121, + "l": 1, + "h": 3 + }, + { + "R": 0.093, + "l": 4, + "h": 1 + }, + { + "R": 0.0806, + "l": 5, + "h": 1 + }, + { + "R": 0.0799, + "l": 4, + "h": 2 + }, + { + "R": 0.0786, + "l": 5, + "h": 4 + } + ], + "lcrit_n_crit_01": 0 +} \ No newline at end of file diff --git a/data/mi1v2_patching/gpt2-large.json b/data/mi1v2_patching/gpt2-large.json new file mode 100644 index 0000000000000000000000000000000000000000..373ae2d54a1421577c7cca284d366f0783e13a0c --- /dev/null +++ b/data/mi1v2_patching/gpt2-large.json @@ -0,0 +1,2170 @@ +{ + "model": "gpt2-large", + "N": 36, + "L_crit": 33, + "N_sem": 3, + "n_heads": 20, + "D_LONG": 200, + "N_PAIRS": 20, + "n_pairs_with_signal": 18, + "mean_gap": 0.3932160536448161, + "head_summary": { + "31": { + "per_head": { + "0": 0.000873485231827525, + "1": 0.002090673040670307, + "2": 0.00695176765508791, + "3": 0.023251998106342847, + "4": -0.01170439900410266, + "5": 0.008606441759426599, + "6": -0.021880303452676303, + "7": 0.008854620353520085, + "8": -0.00021723517676564807, + "9": -0.03022818332732552, + "10": 0.011620335880770915, + "11": -0.016287246098643363, + "12": -0.0098173748975093, + "13": -0.012493868742498887, + "14": -0.013360882390253133, + "15": 0.001128513200349463, + "16": 0.0007929763837034596, + "17": -0.005624446014934569, + "18": 0.015365077177139863, + "19": 0.012962272964243507 + }, + "top": [ + 3, + 0.023251998106342847 + ], + "n_critical_005": 0 + }, + "32": { + "per_head": { + "0": -0.009466131591831236, + "1": 0.009789887894650254, + "2": 0.006810716379191615, + "3": 2.283531808145822e-05, + "4": -0.026277986521184387, + "5": -0.02375178209997704, + "6": -0.003060464741043329, + "7": -0.0022547564901130765, + "8": 0.002259166576713317, + "9": -0.007468298836170942, + "10": 0.01642431178261651, + "11": -0.029831112532696005, + "12": -0.011728154625311911, + "13": 0.0029336147852254083, + "14": -0.003109467998265064, + "15": 0.021364538030361525, + "16": -0.01952892166624336, + "17": -0.00120770437995368, + "18": 0.01913491423451392, + "19": 0.0031533453476405263 + }, + "top": [ + 15, + 0.021364538030361525 + ], + "n_critical_005": 0 + }, + "33": { + "per_head": { + "0": 0.0006450908665992998, + "1": 0.0025598676807161405, + "2": 0.008142171192839031, + "3": 0.013329656592229629, + "4": -0.010896071583085906, + "5": -0.0034173779124828502, + "6": -0.015706286182569296, + "7": 0.0026874442666631024, + "8": -0.01019039438092096, + "9": -0.008890727233177905, + "10": -0.00022458257213978008, + "11": -0.04999697725146695, + "12": -0.01878227888079755, + "13": -0.0017251781424580575, + "14": 0.003692909870480177, + "15": 0.0005470851184933711, + "16": -0.015374705926628207, + "17": 0.0011183220191385364, + "18": -0.01037774055479895, + "19": -0.038594761625979884 + }, + "top": [ + 3, + 0.013329656592229629 + ], + "n_critical_005": 0 + }, + "34": { + "per_head": { + "0": -0.009895841175984093, + "1": -0.0007106507846339202, + "2": -0.011562193725063426, + "3": -0.002077493117390591, + "4": -0.011564123572423686, + "5": 0.0016033313623617087, + "6": -0.004909236059563576, + "7": -0.0005414520806112347, + "8": 0.012113332968321087, + "9": 0.012568547987759707, + "10": 0.016352575998491426, + "11": -0.01356788117835981, + "12": 0.0001949056281875121, + "13": 3.267370173879691e-05, + "14": -0.04100967209342959, + "15": -0.01974903393650918, + "16": 0.004573738810173381, + "17": -0.0032742716267466798, + "18": 0.0024617750609563283, + "19": -0.001220166169512089 + }, + "top": [ + 10, + 0.016352575998491426 + ], + "n_critical_005": 0 + }, + "35": { + "per_head": { + "0": -0.013965882386325342, + "1": 0.019369249359784332, + "2": 0.03197028273820614, + "3": -0.003729091984949577, + "4": -0.009495523889067874, + "5": 0.0010119895525568091, + "6": 0.012401832105052912, + "7": 0.014255242580387052, + "8": -0.007049148012869698, + "9": -0.003812029012796658, + "10": 0.002680425968203027, + "11": -0.0037779743422534542, + "12": 0.0012023352552257556, + "13": 0.003116026687049234, + "14": 0.0006005194576096066, + "15": -0.012711618176583683, + "16": 0.016743911467974222, + "17": -0.06213005144078118, + "18": -0.005488414617758609, + "19": -0.0018169292943320406 + }, + "top": [ + 2, + 0.03197028273820614 + ], + "n_critical_005": 0 + } + }, + "raw_patching": { + "31": { + "0": [ + 0.005821631332054707, + 0.0008613406309947755, + 0.006299405052082187, + -0.00045769297323552873, + 0.014417141447828068, + -0.008124522163347482, + 0.001714859759036886, + 0.008535659227278914, + 0.00020760479596085234, + -0.0034869467744280806, + -0.001307647230381052, + -0.0007861050427230691, + -0.003212904868691408, + 0.0014991322410476988, + 0.0014163161561130471, + -0.00368734448943843, + -0.003178435739055209, + -0.0008087571882014252 + ], + "1": [ + -0.0024668380650119765, + 0.00279973154665954, + 0.1109982264392159, + 0.002314695223708101, + -0.17386945052058994, + 0.009629760845180226, + 0.0002630589182602188, + 0.001288339223264256, + -0.001245628775765114, + 0.02344668199797373, + 0.012322060440129144, + 0.0054632790180986785, + -0.024888483169432485, + 0.007441095990314887, + 0.002005424310440255, + 0.08219772348796071, + -0.012922586063460232, + -0.007144976114880378 + ], + "2": [ + 0.040897858713023705, + 0.015420743598267624, + -0.07813294330727745, + 0.02325508554771677, + 0.23289451454221657, + -0.09582821737103825, + 0.018386396446290594, + 0.011247432068062757, + 0.01949842166380523, + -0.11780079760572698, + -0.01890285315870361, + 0.009706622688916743, + 0.0368318292981446, + 0.02296395885651321, + -0.03368356969008014, + -0.005325654362157184, + 0.02405269519045398, + 0.019650294673154184 + ], + "3": [ + -0.0007428470805594618, + 0.016471329504130237, + 0.14252234591490462, + -0.013876929760449838, + 0.2655896000933827, + -0.033961313936018485, + 0.004628415021281147, + 0.04675913533847329, + 0.0012665386113295163, + -0.020274201168144698, + -0.021846349019990347, + 0.007907114219359516, + 0.029729618535774126, + 0.0021233394841824013, + 0.026842450858634834, + -0.027073414829423794, + -0.004443779271152732, + -0.003085086601541808 + ], + "4": [ + -0.014349728819624444, + -0.006754408530705118, + 0.1279117903263656, + -0.017202296719209025, + -0.385407813436832, + 0.08745016607456206, + -0.012585591813628092, + -0.006491120894707377, + -0.0025491329892531273, + 0.0029237745197573943, + 0.022348645465979915, + -0.0015574847748475794, + 0.01231613532998373, + -0.013285218664546645, + -0.0011040463164897236, + 0.023507681577035915, + -0.016998935482838838, + -0.008851596924850554 + ], + "5": [ + 0.05173836727394789, + -0.010386020362139902, + -0.08336890040970704, + -0.005099395629288476, + 0.16355078876862617, + -0.01015565270418435, + 0.012793195068038858, + 0.009987923965523724, + 0.011027324530776712, + -0.0539566232019819, + 0.022521450918515477, + -0.0004983951279512254, + -0.017563367523827433, + -0.00407922020532339, + 0.02050351484038104, + 0.053713159398422035, + -0.006735469410485581, + 0.0009232714803361401 + ], + "6": [ + 0.007718687048537204, + -0.006673517410576913, + -0.33220217352077286, + -0.0038446209751784415, + -0.010753436825782494, + 0.02228593517096546, + -0.008206016309620123, + -0.0029967890628103348, + 0.020535512168807512, + -0.08005813967594065, + -0.0007956788374212122, + -0.0031147807636618435, + -0.0019984575738271914, + -0.011788438372418067, + -0.015021076884513158, + 0.0446450913074073, + -0.010430987281368816, + -0.0011465743499988346 + ], + "7": [ + -0.013983630348022557, + 3.445362523979102e-05, + -0.11308448101560442, + 0.001367725785516697, + 0.2753981257587326, + 0.025145743792657316, + 0.0025822432192462563, + 0.020186726985494795, + 0.013520075641990112, + -0.08830001387004338, + -0.002742964160769722, + -0.0037496681936997115, + 0.01439914303193438, + -0.0002874081579165811, + 0.007128822813791155, + 0.015912715640482585, + -0.0011570131502674618, + 0.007012568964599613 + ], + "8": [ + -0.029524843284314956, + -0.005166545802262575, + -0.09497877165623271, + -0.006106855483825628, + 0.11992255936116575, + 0.05113030683149477, + 0.0064186376055493394, + 0.019405815486994465, + 0.006327652292716877, + -0.028306993447339755, + 0.007892953524208313, + -0.0013953175722301162, + -0.0012298200454321179, + -0.0021040535030456093, + -0.011828460279105562, + -0.027972879073269383, + -0.014691610030664538, + 0.00829799189381179 + ], + "9": [ + -0.018790836096947677, + -0.0045798355173820755, + -0.5228912606457382, + -0.006220877242070268, + 0.008243335399602538, + -0.009225562970077882, + -0.0059053172299172365, + -0.012770538988660972, + 0.013058752394850484, + 0.032727221188714614, + -0.0014172626293774914, + -0.0010579569306964649, + 0.004435038538839575, + 0.006973528545193642, + -0.017526794326815892, + -0.02193361915030613, + 0.021285244817516945, + -0.008510559048586854 + ], + "10": [ + -0.001918355991193879, + 0.006949146412495241, + 0.012117887782983906, + -0.00845045763215913, + 0.06920459774996843, + -0.03404099452071608, + 0.017007114550547823, + -0.030766924545345246, + 0.0015331390147756468, + 0.0001981219758197773, + 0.0108300014208482, + 0.0007487254081267272, + -0.00708940013689723, + 0.009968029909018445, + 0.0022591140288634394, + -0.026132648656013863, + 0.17488097413164272, + 0.011867974951111531 + ], + "11": [ + -0.04006714799927979, + -0.0016442867813714758, + -0.23589578531592928, + 0.0020539975886604955, + -0.1575856747558336, + 0.08528285417078747, + -0.00809510498192122, + -0.01718664293745872, + 7.243764463382257e-05, + 0.013914064208211082, + -0.004654140882095875, + 0.000999244474335544, + -0.01158592967800841, + 0.01939934507469349, + -0.004974113678356996, + 0.05926138526989815, + 0.014041627739684167, + -0.006506558936229341 + ], + "12": [ + -0.03059118828705354, + 0.003253121281699109, + -0.004145414937499246, + -0.0033606976444124554, + -0.004672382793304955, + -0.03434233200466334, + -0.005824266644291115, + 0.007706970443005358, + -0.003980523250612709, + -0.11538539564520077, + 0.003444502714346933, + -0.002259013674645592, + -0.039646323714617904, + -0.0002638886687253715, + 0.004825773694027611, + 0.0495141298927143, + 0.0025809434268997144, + -0.0035667623428334533 + ], + "13": [ + 0.023041572169037933, + -0.0013087384312100328, + -0.205035474114546, + 0.0044409601239321006, + -0.029008192925183612, + -0.026855254518896622, + 0.0007500734020662996, + 0.040765557212852625, + 0.004235175176593467, + 0.016059683052727394, + -0.04079240354182785, + 0.005673586659211431, + 0.02628227922092222, + -0.004439538779732721, + 0.01954922576921748, + -0.01908837511365171, + -0.03731144048000544, + -0.0018483322464868852 + ], + "14": [ + 0.002863555681511474, + 0.0018919534207763504, + -0.22862098851384727, + -0.0019249870546841654, + 0.0924621656987767, + -0.07635428247094608, + -0.0017802690035772647, + -0.025369738745583476, + -0.0065514995502322215, + -0.0003684225678010327, + 0.0032768556335288496, + 0.0005044362810173008, + 0.004349207348168792, + 0.012931485547110853, + 0.019068774291309886, + -0.02244989326986036, + -0.012795269944608407, + -0.0016289658056163217 + ], + "15": [ + 0.001825167289331581, + -0.0021026697954313043, + 0.0005012429826387976, + 0.0011600241789489951, + 0.004550645772635719, + -0.00019847709279219062, + -0.0005609553689386829, + 0.00045965043899070005, + -5.750204780210658e-05, + 0.013508546461958688, + 0.0005687104818521144, + -2.945062119711786e-05, + 0.0008070694048148274, + -0.0002610663300224264, + 0.000889567485644052, + 0.001498342222528497, + -0.001729935685100674, + -0.0005156721717691387 + ], + "16": [ + 0.016769972533705272, + 0.002535087758585783, + -0.12503980350692168, + -0.005192004850773559, + 0.02501985624802008, + -0.0009156023550705436, + 2.2751041579262168e-05, + -0.005181364267475812, + -0.0007592884039323618, + 0.04369643407123003, + 0.0006680091374135946, + 0.0025082112386212046, + 0.002034327325152295, + -0.002587143811033054, + 0.025270708413208512, + 0.028167915962878758, + 0.01316158298832243, + -0.005906074616847929 + ], + "17": [ + 0.09863491585256468, + 0.0028896105690242122, + 0.03468059555554924, + -0.00011134519114969588, + -0.16260008060720926, + 0.012289643636176299, + 0.008156248406165488, + 0.01473682142212119, + -0.013652255673950798, + -0.042751350092957816, + -0.005055849079594591, + 0.002849347600821153, + -0.02151800760742009, + 0.0034211448977533457, + -0.012165295976006364, + -0.04255246020948082, + 0.019808824562059818, + 0.0016994636667117557 + ], + "18": [ + 0.1353006762338706, + 0.02044398229264875, + 0.04329655601388101, + 0.029891366002922443, + 0.11988777735526025, + -0.10802948799509539, + 0.027169009465935142, + -0.005901318539299955, + -0.0005921964143775391, + -0.03527920085172085, + -0.009452716172281175, + 0.010710209242018529, + -0.011985621192773849, + 0.0060656762624129515, + 0.045061340972210435, + 0.0032008995412362226, + -0.03931722775595173, + 0.04610166472762168 + ], + "19": [ + -0.04471193915353062, + 0.0014165932580476395, + 0.2754330189600193, + -0.010953583179351332, + 0.1622754485520913, + -0.06677088123868083, + -0.017690356767975042, + -0.003271096582853172, + 0.019140154034804444, + -0.04786205399687054, + -0.0008846607495477335, + 0.0054604472275989555, + 0.006871619503851959, + -0.00955220534011786, + -0.015323898317491298, + 0.009598109520423741, + -0.024539623504659207, + -0.005314178869376621 + ] + }, + "32": { + "0": [ + -0.00799026555110733, + 0.0007924333805151934, + -0.1001808609895651, + -0.002169625287162103, + -0.013930193365151123, + 0.004934401299636505, + 0.010011880234974058, + 0.006640218976172102, + -0.012319253656720146, + -0.08110186310600398, + -0.002332228812765417, + -0.0009673396347053329, + -0.0008339717183086549, + 0.005329516250728091, + -0.005727624108310401, + 0.00545644380577759, + 0.00045454088046221715, + 0.023543422748571574 + ], + "1": [ + 0.040185630777361855, + -0.0035766857622119286, + 0.15898885524997106, + 0.001359696084231863, + -0.012637462145663525, + -0.04227417202683301, + 0.0008190374968534381, + -0.023799678285518468, + -0.00799203786465123, + -0.013202511239734947, + 0.0084971278116181, + 0.003993202176675878, + 0.04079159363192656, + 0.012225430481590742, + 0.01118266168490725, + 0.012773768993592869, + -0.012192193592152395, + 0.0010757186317404796 + ], + "2": [ + 0.041650024663769394, + 0.0022749379216302594, + -0.045484413897561166, + -0.007113780024943791, + 0.20927753253238482, + -0.05447834012669734, + 0.007608801468164492, + 0.0038452477584383295, + -0.0030237115785809033, + -0.05700432934065635, + -0.001381154027355135, + 0.00016009055625099966, + 0.0053023178833786835, + 0.0038689559719539764, + 0.016339602031848553, + -0.011539300209947645, + 0.0055873907246989495, + 0.006703022518672962 + ], + "3": [ + -0.015699633729458304, + -0.0046282703238785935, + 0.2121951205016975, + -0.011047798341093382, + -0.1632725327213822, + 0.012324413345862522, + 0.008547992903358408, + -0.0028921732818930966, + -0.005213270074110468, + -0.044252018675762936, + 0.0024992310971188154, + 0.0004462901827563245, + -0.001834481567769576, + -0.00656711177196954, + 0.012375995444193962, + 0.02828952719992931, + -0.017277020689804665, + -0.0035832237723278184 + ], + "4": [ + 0.10371902517273776, + 0.029269602960957534, + -0.42297456825513136, + 0.04718787788387352, + -0.046978895976356516, + -0.19823225608648345, + 0.015881648962423698, + 0.016739348693064544, + 0.016988494616235358, + -0.10619928054506335, + -0.030206135184630556, + 0.008282987211689399, + 0.13419386395740127, + 0.02755120002836673, + 0.07230062490994299, + -0.14188245353589335, + -0.03931387733177142, + 0.04066903513731821 + ], + "5": [ + 0.061398041855559825, + 0.06421107156102791, + -0.1823508423732307, + 0.06077306183097439, + -0.4252390071996091, + -0.23385527239792267, + 0.046847238491898216, + -0.00460638934558422, + -0.0014226155982209486, + 0.08023687099029721, + -0.068901660623303, + 0.07785215077648112, + 0.027294318633309066, + 0.03489257338451089, + 0.13570368528211643, + -0.2463740407820269, + 0.07574527306041512, + 0.07026346465372071 + ], + "6": [ + -0.019365943514155003, + -0.0021246402231204463, + -0.0008263735659720718, + 0.0007547919207743807, + -0.06241630959741246, + 0.025954139542862006, + -0.002943416004317043, + -0.0021087905444734626, + 0.004421310052108727, + 0.024324320197243465, + -0.0026933148329889816, + -0.0032856321238117904, + -0.009153191900638003, + -0.0007229890977377825, + -0.015160913302861878, + 0.01483427636852486, + -0.0014082949637908005, + -0.0031673937490136342 + ], + "7": [ + 0.022341325637901234, + -0.011041138569597956, + -0.07874256315102733, + -0.0042391469649732776, + -0.011367918930112924, + 0.02722178520850556, + -0.0068310002341734665, + 0.034770331594619085, + -0.0010790968710914807, + -0.01641208724801534, + 0.013978542557904748, + -0.0006022274462743973, + -0.009360724033304672, + -0.002726849576828839, + -0.0011971131219447838, + 0.01041038080185573, + -0.008631809496541711, + 0.002923693021064444 + ], + "8": [ + -0.08873694444761916, + -0.01412448836460824, + 0.16616204874476143, + -0.015284268738971716, + -0.018596779157471828, + 0.04181492211139414, + -0.002040484041640076, + -0.028460434887327395, + -5.563509819814208e-05, + -0.0012308854667952122, + -0.001544932329385109, + -0.0026348866669754743, + -0.03089666651638731, + 0.002064070371420553, + -0.009169678649150848, + 0.028340007336063504, + -0.010769380123580245, + 0.025829414305310824 + ], + "9": [ + -0.0015123195045081517, + 0.0027123491638049975, + 0.0029261752499994674, + -0.0007762044575339376, + -0.1490524893070186, + 0.00416657021073241, + -0.0006178329728868383, + -0.0014390847973418512, + 0.0015600230890727357, + 0.016097621303416288, + -0.0024025116274160752, + 0.0004857464637191299, + -0.004801422427374561, + 0.0016030883832728452, + -0.0007275273116791503, + 0.0011403921663042313, + -0.0004277374870197277, + -0.0033642151886201757 + ], + "10": [ + 0.076878016502057, + 0.0045823321568922055, + 0.3236539486006355, + 0.006368088432292222, + -0.1066126451013293, + -0.016970515802684095, + 0.013817702909155008, + -0.001519811935372476, + 0.004199889829078538, + 0.004626780439569948, + 0.00039332584345781155, + 0.004141021640761413, + -0.0034358097519259796, + 0.0016553016492773303, + -0.0018315736281688739, + -0.008056170816688442, + -0.013619474292964959, + 0.0073672054130543094 + ], + "11": [ + -0.03657257167944361, + -0.0012553103456932554, + -0.39348928847909503, + 0.00036990157252134544, + -0.050578833587575346, + 0.015032104487677151, + 0.001589729030350944, + 0.010068651103554349, + -0.03460017701027407, + -0.005114076192862847, + -0.005648417030640047, + -0.0018827631102490798, + -0.006067112224131781, + 0.0028185755846745564, + -0.007912568145009149, + -0.02514828600139713, + 0.006601452443273134, + -0.005171036004208227 + ], + "12": [ + -0.017667246605922254, + 0.008198964151266211, + -0.21159227421176705, + -0.0018479019223497605, + 0.02115905359251003, + -0.008864827232083317, + 0.002597173590282647, + -0.008494471911957166, + 0.010021598779121036, + 0.02408320153730961, + -0.002957036587045121, + -0.008808378742404649, + 0.008042510672107121, + -0.0027781220632656757, + -0.003818573545651021, + -0.004761194658111227, + -0.0028914160676085507, + -0.010727842030045275 + ], + "13": [ + -0.0248188138402689, + -0.0004923373113975934, + 0.03155121369096648, + -0.003978449329925672, + -0.048202063184033125, + 0.04033286323601888, + -0.008561501334296095, + 0.04273925336307075, + 0.00043761298716927865, + 0.03123835561723493, + -0.0028577379314836406, + 0.00044987711738930683, + -0.005265167069506254, + -0.005067509141138017, + -0.015102805601993997, + 0.033495864876678404, + -0.008752424767032913, + -0.0043411652433944636 + ], + "14": [ + -0.02893242939390463, + 0.00279973154665954, + -0.08261703593574886, + -0.006522258696961032, + 0.0466948429281283, + -0.010597517764780105, + -0.0013607966744596184, + -0.0008220988138220764, + 0.0025022725541936183, + -0.011335106233603854, + 0.010621732162754966, + -0.002685858895970486, + 0.013808573197617498, + -0.0006519602403803296, + -0.0032039546933818166, + 0.009811502823172823, + 0.003737956577167176, + 0.002781981584547734 + ], + "15": [ + -0.009555835742393938, + 0.001431073767206392, + 0.4848171146266479, + 0.0066046969634853256, + -0.03824861249407704, + -0.053220835626378936, + -0.00701514147695562, + -0.02987892602700837, + 0.010442969304735823, + -0.04221768536660071, + -0.004971380742720864, + 0.010880305458035217, + 0.008478071938197662, + -0.002322314362740034, + 0.018743276682383306, + 0.004885100446804242, + 0.004638103873610781, + 0.021071703324276336 + ], + "16": [ + -0.03390204973750404, + 0.01574330942297639, + -0.14966167164059777, + 0.005052288048417451, + -0.087865143918267, + -0.0799283188791091, + 0.014660202417637061, + -0.014891685727730959, + -0.021440422641849102, + -0.08005392431475299, + -0.022912842372579234, + 0.011342264881556674, + 0.05002036822285674, + 0.013193022266917103, + 0.011566739415033972, + -0.03335589722648815, + 0.05258937474222443, + -0.0016762029511218916 + ], + "17": [ + 0.0011022892163140403, + -0.0002092183909488759, + -0.0008399206736109582, + -0.0019276636217791098, + -0.016764926846449042, + 0.0008011520606867256, + -0.0007678476533000983, + -0.00047612536511939895, + -0.00012583240330720725, + -0.0012831559455221322, + -0.0007002489606478415, + -0.00029941464883736493, + -0.00033435732485185703, + -0.0004741529020947852, + -0.0003939985571041632, + 0.0026433234921176547, + -0.0007449109760891864, + -0.0009436693386226363 + ], + "18": [ + -0.04441639784191019, + 0.02365116540736147, + 0.22514615540447291, + -0.0028318079864514, + 0.079262394457641, + -0.034916032214486026, + 0.0042388034342362825, + 0.014236807414115177, + 0.009767693632981864, + 0.029718296372966593, + -0.0032175343587778353, + 0.015754949624258167, + 0.019314580026020878, + -0.012937130224516744, + 0.008992048604221394, + 0.017610683855905383, + 0.017675721167261696, + -0.02262194055405004 + ], + "19": [ + -0.03801433402397031, + -0.0031242946829764117, + -0.07375045398609768, + 0.004278224844559469, + 0.1598870841465806, + -0.01234614441441641, + -0.0027926903538544314, + 0.009334693144520812, + -0.0029303640983826783, + -0.019496888565141146, + 0.02238991243971975, + 0.0016558422982046203, + 0.014264631464465242, + 0.009280320045067477, + -0.02030698798216122, + -0.00012849489197794156, + 0.010025585955551163, + -0.0014654250821614318 + ] + }, + "33": { + "0": [ + 0.007400514195035929, + -0.00011334743375989219, + 0.027216139246522823, + -0.0014212571274155891, + 0.001646348279526808, + -0.008517130135221084, + 7.251894503389817e-05, + -0.0007380766905657119, + 0.004424297171475071, + 0.01447049188498152, + -0.003505113582027317, + -0.0006126106781067145, + -0.0028747043561975754, + 0.0005875168399964154, + -0.0007776038668986243, + -0.018115485217247297, + -0.006427230385896952, + -0.0011036314904483164 + ], + "1": [ + -0.020188666624882147, + -0.01288216054436766, + 0.010620932388886955, + 0.007837523767416815, + 0.20371241158750547, + 0.02547170982096566, + -0.0006882190077726806, + 0.020152129640624527, + -0.0020338548985589255, + -0.19339234056722857, + 0.00010961539899643928, + -0.003055690735234293, + 0.026603825920300826, + -0.0011054159919868503, + -0.019796301602989036, + -0.005364661740079059, + 0.005700188338769426, + 0.004376593102523641 + ], + "2": [ + 0.03961185462732399, + 0.007203304314626453, + 0.20618697826385135, + 0.0018746675932992066, + -0.09395779195271303, + -0.023847674631038318, + 0.0021066042562298067, + -0.009710321460255147, + -0.002630158602065187, + 0.012883829933948241, + -0.00904584960306498, + 9.873509542367078e-05, + -0.0010312553505967238, + 0.0018227604123187426, + 0.0059170646615463365, + 0.003072404649258281, + 0.0007158739665264894, + 0.005288055296483389 + ], + "3": [ + -0.010984285415226022, + -0.007047014681292328, + 0.08677599798088699, + -0.033535244445980036, + 0.09001003428243924, + 0.0334470119802431, + -0.01078825952886638, + -0.002720010303848193, + 0.007576828272729524, + 0.008919704273077632, + 0.02178638294877465, + -0.01043628070767867, + -0.03161278048034206, + -0.007685698677903467, + -0.011315884218604343, + 0.09890664854837805, + 0.03917874355649887, + -0.020542074723153277 + ], + "4": [ + -0.053978889920151436, + -0.004190859081703855, + -0.12474176713886619, + -0.0029586772667517746, + 0.14484966359343784, + -0.07404644299052303, + -0.054966516455497404, + 0.021234532287280047, + -0.004776030476861982, + -0.053793067187900896, + 0.009114842824786269, + -0.0003417027202998931, + -0.11566585633543801, + 0.0032880245889310997, + 0.01615488568193521, + 0.0926792353907328, + 0.006799127469911494, + -0.0007897907585666129 + ], + "5": [ + -0.004654110024437059, + 0.001052583217470717, + -0.2674673196683541, + 0.003791089633279549, + 0.14588152976863422, + -0.02396502240122932, + 0.0005282507466684935, + 0.035669862561246046, + -0.012697497646483353, + -0.015895283966408855, + 0.005984355988740899, + 0.0037085128384370726, + 0.017013791691024956, + 0.0004407552274432676, + 0.0014975724532616276, + 0.015820933574784053, + 0.03676643814667482, + -0.004989244565444367 + ], + "6": [ + 0.022192223714921556, + 0.0053423092237757114, + -0.12680770105379635, + 0.0012922465934392588, + -0.09232303767515472, + -0.04807057237910713, + -0.0072668248744262075, + -0.05105744356545084, + 0.0003360509287136099, + 0.03839941120282296, + -0.006204876379663148, + 0.0016477244987720815, + -0.02781058683988109, + -0.014313490731986327, + -0.00025274487775866583, + 0.013283159458219708, + 0.015317022544322625, + -0.006416021074010333 + ], + "7": [ + -0.039622504764679684, + -0.00884858975180198, + 0.13823468634719707, + -0.02181455713721758, + -0.013559185302159167, + 0.05889843947055846, + 0.00043298076005533314, + 0.032492673057326456, + -0.0014565940810131023, + -0.005769986393661939, + -0.009670657377344684, + -0.0010128370687342136, + -0.020326619438407724, + -0.002093704927801477, + -0.019573791626494957, + -0.014760850715966035, + 0.015120464325744369, + -0.03829536857566332 + ], + "8": [ + -0.0188693808599459, + 0.00010835415473963263, + -0.011643739015622882, + -0.008911362485908592, + -0.1931966518020772, + -0.06793132029945852, + 0.008360296810329496, + 0.013721965972593335, + -0.015461703230113191, + -0.018538315431068438, + -0.0028474211880486814, + 0.008727389534112574, + 0.005140904002415718, + 0.016449060350548158, + 0.006429168301715297, + 0.0944460401554295, + -6.812529166632739e-05, + 0.0006577414654487696 + ], + "9": [ + -0.028490448693643446, + 0.0024556946221636557, + -0.06465357120658545, + -0.0024121222659640846, + -0.08922164214858133, + 0.022511938283925913, + -0.006449920287720825, + 0.0013855412874235796, + 0.002196279514103837, + 0.0025039245454669726, + -0.0019769459607240168, + -0.0018406638248198665, + 0.00709196226199188, + 0.0021445070244544896, + 0.004066594220087095, + -0.013177610082666399, + 0.007829941309387235, + -0.003996548795501555 + ], + "10": [ + 0.0004765936466671816, + -0.0006496256005357698, + -0.0020252925920135204, + 0.00016112933911566568, + -0.005431789922241616, + 0.0028815396902457453, + 0.00017205475194317015, + 1.6474926128698926e-05, + 0.00010081527861408297, + 0.0010850339697023549, + 4.5135752527945585e-05, + -4.360957369573222e-05, + 0.0, + -0.0005658789099405025, + -0.0003406150595588749, + -0.0011151520982371356, + 0.00025239862158344246, + 0.0009383014811788215 + ], + "11": [ + 0.004836493626653271, + 0.0034868067398472562, + -0.48586701546866157, + 0.015744638279302187, + -0.48754517377832085, + -0.062292832378676, + 0.008602737597158508, + -0.012196387813075815, + 0.004187567961692372, + -0.002750101638826015, + -0.01976043245673458, + 0.008849911669733918, + 0.04005370160466729, + 0.02759165354977561, + 0.031684286843959256, + -0.03071716283765542, + -0.005969339081254424, + 0.062115057054009895 + ], + "12": [ + 0.007284693951292787, + -0.0005447667411103189, + -0.23289510097391594, + 0.006401813177688524, + 0.003385448574801605, + -0.08122638804072811, + -0.0022466653559521394, + -0.0371443684497646, + 0.0026200770742037783, + 0.0034329901512260984, + -0.034466305426803935, + 0.009900694731164417, + 0.01268892453125534, + 0.013345428556876142, + 0.01727735639138304, + -0.04369514692742752, + 0.008627342264301296, + 0.009172952657153597 + ], + "13": [ + 0.011361034024183597, + 0.001200883604372426, + 0.1780089943749676, + 0.005372405472972826, + -0.16643189825779808, + 0.0164967785082093, + 0.0016814441667173446, + 0.02336721147464012, + 0.0010014317675665575, + -0.01629237099028594, + 0.003569593228495811, + -0.0017806298662257415, + -0.02150647804449416, + -0.0027974080444024677, + -0.004764359050833916, + -0.027353350129804308, + -0.03292796884409831, + 0.00074148004157228 + ], + "14": [ + -0.01585805452262421, + 0.0015174574942568827, + -0.06638082743054348, + -0.0003715075127783122, + 0.15272199093004843, + -0.011740571970714691, + 0.0055327689240568185, + 0.026514746111528053, + 0.00036293500301069867, + -0.005593784296017967, + 0.011917128260307004, + -0.002251084661246368, + -0.009162159338469278, + -0.0048577152975524274, + 0.007255903883168877, + -0.011364914285120438, + -0.014521855205528771, + 0.0027519215828623713 + ], + "15": [ + 0.024557885475054466, + -0.00633297578139521, + -0.2435295804704418, + 0.0003640131249124673, + 0.09537805719385412, + 0.04658561602792468, + -0.002297855199505479, + 0.016189086160366, + 0.006514907337994517, + 0.04690516700727289, + 0.0016029640112067533, + 0.000335283995167188, + 0.011934378690880845, + -0.008974566685581753, + 0.012891878447020996, + -0.002826887623514714, + 0.006918625932342592, + 0.0036315344893221514 + ], + "16": [ + 0.018468669441938017, + 0.0011479548467576747, + -0.15757318250170743, + 0.005815644983895654, + -0.04204564813876034, + -0.0640023431049153, + -0.0004123626286241268, + -0.028303923089104756, + 0.012891847100256057, + 0.007132391129511983, + -0.013160295844219563, + 0.004293560755679817, + 0.002596713783428024, + 0.004372273040645861, + 0.015347046913771999, + -0.050654522059018534, + -0.0038630390798987937, + 0.011204507771056027 + ], + "17": [ + -0.00318305980218221, + 0.0021486079624176923, + 0.0040912265069437, + -5.995510292675932e-05, + 0.00933896858562566, + 0.007721773026148729, + 3.839238266500491e-05, + -0.0005140176952154065, + 0.0005081836821991367, + -0.0002596662491595379, + 0.0002617873646620844, + -0.0014536524565244074, + -0.001334867174312778, + -0.00026624061764449246, + -0.00029762480932328875, + 0.004921813273083654, + 0.001685263362696525, + -0.0032171358946596514 + ], + "18": [ + -0.00024761569351982064, + -0.0009801806716769532, + -0.0075592860624986245, + 0.0011905370438313636, + -0.1898343912312126, + 0.002951079109618192, + 0.0006512485652063796, + -0.00628518431809864, + -0.00011892468977253861, + 0.0032255943807934805, + -0.0034831905022280293, + 0.0038189526679262642, + 0.006815252751769653, + 0.0029291171838732415, + 0.0013095491610224707, + 0.0022486606096139773, + -0.003377227573753672, + -5.33207172752267e-05 + ], + "19": [ + -0.01985718109968626, + -0.005332322665735193, + -0.16947431656246917, + -0.0013382835474723062, + -0.5060086219131549, + 0.04676815700377735, + -0.0029910509976236233, + -0.021629106768062387, + -0.013899439801515697, + 0.002423832682901531, + 0.013447875067469044, + 0.002774588331628469, + 0.00031257926154732997, + 0.0026299492813610555, + -0.016776590839188303, + 0.0014203274666847467, + 0.0030689885491650437, + -0.010245092717264866 + ] + }, + "34": { + "0": [ + -0.03738198211847615, + -0.004107970649967547, + -0.1273902266822685, + -0.0015422379601070858, + 0.004597021780509713, + 0.004183955065575522, + -0.0015129442650209342, + 0.00031796607428388925, + 0.003955319430959188, + -0.0031176811343895166, + 0.00028113125860263253, + 0.003583725270415951, + 0.000584164521580256, + -0.0031605489575147437, + -0.0011985303829415612, + 0.013494258209326326, + -0.028354639837973542, + -0.0013559207903076104 + ], + "1": [ + 0.027689025857627683, + -0.0013566739098045246, + -0.019819418475690836, + 0.004556587822433708, + 0.003495591593502342, + -0.01696327211316613, + 0.012133414862240255, + 0.007749805250939975, + 0.0032272090854130334, + -0.046100033020430815, + -0.003770769725477511, + 0.0033547278120050284, + -0.00047014995486832005, + -0.0013123874968694947, + 0.004749714020533881, + 0.008616041417449474, + 0.0018751207329141587, + -0.00044624788216246773 + ], + "2": [ + 0.010474410089322305, + 0.002555060874666821, + -0.06778972662498767, + -0.00946969438191404, + 0.07254946731788028, + -0.011330579143997975, + -0.017452892771491493, + -0.02225597770725938, + -0.003926195017137342, + -0.05848054882857307, + 0.005682591243268349, + -0.0043805911170379675, + -0.01658719786276569, + 0.0016289598213831756, + 0.00934352933142223, + -0.08419857252018867, + -0.008091274395451507, + -0.006390255358280021 + ], + "3": [ + 0.01589666127053859, + 0.0031772234405911632, + -0.0180582944826356, + -0.0019785183965830575, + -0.027819807723412502, + -0.015797038100774062, + -0.009316551526707859, + 0.025074837567879767, + -0.009698616497635178, + -0.033042530205551195, + -0.0013205431596747508, + 0.00159637469771044, + 0.016883123311197793, + -0.0025048255988638205, + 0.006634671146248043, + 0.008336106117068958, + 0.00037524750819485246, + 0.0041676045193777865 + ], + "4": [ + 0.018809473837320137, + 0.011216402663209068, + 0.018749196972218808, + 0.0009812294970066948, + -0.16676232731390028, + -0.05365835447326391, + -0.0031737703003070724, + -0.00012191445335237205, + -0.00372120395062204, + -0.015054740945590481, + -0.0011219458485517903, + 0.0005125540804498397, + 0.017770899656494105, + 5.8328333194199764e-05, + 0.004525314362710767, + -0.03896378144066831, + 0.0004076349419378606, + 0.0013927800780884718 + ], + "5": [ + -0.003262935832349894, + 0.0009701941136364341, + -0.03584564681249348, + -0.0039945087324953395, + 0.06193515851571977, + -0.0027583969684403715, + -0.007096192062581741, + -0.01669074766098488, + 0.00327052231622501, + 0.01737234652656311, + 0.007582806424694858, + -0.0014836694358214698, + 0.011365586919868489, + 0.007872913811865495, + 0.00175220701234933, + -0.006858414859322631, + -0.006563480969229607, + 0.0012922222153076753 + ], + "6": [ + -0.02109259703294644, + -0.005312349549654155, + -0.04350653618228375, + -0.0012440683857302558, + -0.08340145316039502, + 0.042421943292999455, + -0.009135965134172464, + 0.004795850996064257, + -0.008897135032653217, + 0.01744147845004065, + 0.00961262569552304, + -0.001970171043673859, + -0.018899515760687536, + -0.0022884462983046925, + 0.00840624739222, + 0.02286061801386128, + 0.0016428246564125833, + 0.00020040001123575136 + ], + "7": [ + 0.00048458124968395004, + 1.098521384457105e-05, + 0.011596324138886779, + -0.0006316698344069285, + -0.027298077634830064, + 0.009053163159550359, + 0.00198360643769192, + 0.0014736821422121188, + -8.699985154474567e-05, + 0.0016802429693992176, + -0.00016313350556528905, + -0.0009854630939035592, + -0.002640269910037078, + -0.0005348331842081059, + -0.0009410613018603036, + 0.0012734761615670993, + -0.0018807047732146774, + -0.0021399858342674876 + ], + "8": [ + 0.005359681624251601, + -0.01329959867046136, + 0.06986920764755673, + 0.010493748952439848, + 0.14038017583458162, + 0.006898889896908115, + -0.005947975432878353, + -0.008749833266951999, + -0.007666815243640613, + 0.010027501193193324, + 0.013308599031097098, + 0.0023111186198404928, + -0.0011042759157942558, + -0.0024954178031873367, + 0.010559539266657382, + 0.0062067621928630695, + -0.0019823343066841166, + -0.016128980190011695 + ], + "9": [ + -0.008838282738054242, + 0.005261418103647507, + 0.07250412008332013, + 0.000842583321488564, + 0.21687160382175144, + -0.04570478338254036, + 0.012623984196293096, + 0.016861263146416915, + -0.00471255419032719, + -0.04082998846362474, + 0.014536291499857218, + 0.001972814048140267, + 0.00813474717551453, + 0.005326223522241322, + 0.0009552339118280792, + -0.016247720180282216, + -0.007932687650916778, + -0.0053904024450787905 + ], + "10": [ + 0.016937712197057406, + 2.5465723003323797e-05, + 0.16122412801038732, + 0.009369055459144121, + 0.0201213904163294, + -0.05233131055357306, + -0.0019181971931515416, + 0.014346365672871024, + -0.0023596376044507307, + 0.05977550778542064, + 0.009749322546036247, + 0.0009201431263766184, + 0.001740964001814842, + 0.007396879350635413, + 0.02414398592077035, + 0.024810986909955035, + 0.0013301183995835396, + -0.0009365121953642166 + ], + "11": [ + 0.037475170820338445, + 0.0024611872290859413, + -0.22133264460412638, + 4.6572267452036254e-05, + 0.048004965150568644, + -0.04495723462428656, + -0.003537786965575267, + -0.004858455715353313, + -0.012377502484363838, + -0.07957253006712281, + 0.00024502265658027606, + 0.0010785346083277843, + 0.03819359878595121, + -0.005477689032632712, + -0.00036423607617183434, + 0.011089568088024849, + -0.02027788394730338, + 0.009939482700130346 + ], + "12": [ + 0.0029154751011204686, + -0.004402074784260835, + -0.020063266413190794, + 0.0020331203653199278, + -0.008417245429130017, + 0.00904302199422521, + 0.004527457274273171, + 0.027384622211123356, + 0.007853510204037063, + 0.015807604453705635, + -0.005531064074067389, + 6.343210719379232e-05, + -3.0745501135802944e-05, + -0.0028195163642422047, + -0.0015495386898101383, + -0.02048346251226936, + -0.002751815060095585, + -7.121357542127593e-05 + ], + "13": [ + -0.006183736002148208, + 0.00017326678200300702, + 0.0062248959600683116, + 1.070626837977845e-06, + -0.006573799116138732, + 0.00575728442887712, + -0.0004038309880319035, + 0.000276778758962142, + 0.0008905349610910662, + 0.0077714398855604555, + 0.001615859940500452, + -0.00014970732441868246, + -0.0003305141372098817, + -0.000945013075702801, + -0.0019515683925627077, + 0.001631426217791365, + -0.005390832506120693, + -0.0018254293880599422 + ], + "14": [ + 0.008063485245427706, + -0.0012777801012844234, + -0.6677978975050868, + -0.0007949404271985499, + -0.008121598378933302, + -0.030734974624717692, + 0.009956424571124606, + 0.013012720402752847, + -0.011505823714272813, + -0.038349669940808635, + -0.011578610116347412, + 0.00111761331722396, + 0.005958221907609146, + 0.0020687742692587947, + 0.0016770921795201193, + -0.034016728099517564, + 0.01977085308801629, + 0.004378740245501167 + ], + "15": [ + 0.022710086643842043, + 0.008062148306111098, + -0.08075430863540196, + 0.00020502503947275732, + -0.20442544270856813, + -0.05250226162619699, + 0.00482322081480358, + -0.015216241772466327, + -0.005931672281716007, + -0.043134947961034405, + -0.019143362240031093, + 0.011990933692026527, + -0.016196473785831528, + 0.008092585840911394, + 0.009494231417412912, + -0.05178114691546798, + 0.060163567005847914, + 0.00806144830912102 + ], + "16": [ + 0.01201868000589753, + 0.004886422849226013, + 0.14901818402775066, + 0.0005984804024296153, + -0.06690318835922143, + -0.03195915815325347, + 0.0012228684848853415, + -0.0017751732903673093, + 0.004711807410485603, + -0.05567564749430754, + 0.0033200569966627402, + 0.003258258148981136, + 0.02487951573160121, + 0.000866457981804161, + 0.002155181555766418, + 0.016931496569736262, + 0.004612417288228395, + 0.010160638426815515 + ], + "17": [ + 0.00794233993300672, + 0.003950183032927344, + -0.020327435012149078, + 0.004111742371253914, + -0.10037507204227704, + -0.009200934425716806, + 0.009568234924178446, + -0.01228370492155792, + -0.0032546532445913118, + 0.04016480446821281, + -0.010456019471330937, + 0.0026309221602758624, + -0.0041147729020082945, + 0.0046582700292109695, + 0.011028652656590756, + 0.001099090236739893, + 0.0071185345751011595, + 0.008802928350693299 + ], + "18": [ + 0.0020328449677675595, + -0.003595160894586889, + 0.011603097692706222, + -0.0028773096270654584, + 0.051749827786393705, + 0.026635046357550544, + 0.0061043888437357805, + -0.03939649085155774, + 0.0018561212962615052, + -0.036413976083437535, + 0.005300226939710182, + -0.0013198031589041728, + 0.004452973414502127, + 0.014011500490771196, + -6.944578884210071e-05, + 0.002884251414576295, + 0.003043301963782658, + -0.0016894436661499682 + ], + "19": [ + -0.010940353598633795, + -0.0005802190221541618, + -0.0609145694982528, + 0.003232222423855114, + 0.010834594839561985, + -0.016075195778263846, + -0.0020490156822322993, + 0.015983973330063697, + 0.0055567888012399355, + 0.0016018372513088377, + 0.001287658539975819, + -0.0001572587657512768, + -0.0008531876565185317, + -0.0017832476704775105, + 0.0031538781381623424, + 0.018732719609069907, + 0.006150261986991228, + 0.004856121700837761 + ] + }, + "35": { + "0": [ + 0.03251486934692526, + 0.01063568431315288, + -0.31717165759542837, + 0.006991193251995328, + 0.041013781963563964, + -0.03799315152171678, + 0.003318808190374869, + -1.1532448290089248e-05, + -0.0015245510465974102, + -0.034581980111282745, + 0.004962353592215274, + 0.0034049448968667805, + 0.026202853342988063, + 0.010731942917948933, + 0.006498614090557397, + -0.027435953988932983, + 0.0013904260348291408, + 0.01966747181697439 + ], + "1": [ + -0.015185764602046204, + -0.011975880402190548, + 0.2371963076492624, + -0.012871075846169653, + 0.0411123309802962, + 0.06676798376287364, + -0.0003625947251694908, + 0.016149546337657122, + 0.010284651978319634, + 0.05470695749338446, + 0.0037340163269904696, + -0.004021708867706422, + -0.02470913441280697, + -0.007165447576993911, + -0.025579198890173762, + 0.03234629450380432, + -0.004005990511592071, + -0.007774804721621311 + ], + "2": [ + -0.03813681060356076, + -0.009243058794402487, + 0.22295829752079274, + -0.003957036793166115, + 0.2762560819044015, + 0.07385231211144161, + -0.011273141102524404, + -0.008379147429056273, + -0.0009678266746951965, + 0.04306160067636921, + 0.025085161662102787, + -0.0011127048803577736, + 0.008601053942740874, + -0.004193524922792668, + -0.0008739776146794988, + 0.014774618025820816, + -0.011615920633138871, + 0.0006291128924150909 + ], + "3": [ + -0.0123128900503485, + -0.00680234400929961, + -0.1481782633541397, + -0.006367017805454244, + 0.10517498885723547, + -0.006177418420918983, + 0.0018357246674267163, + 0.002036300869507187, + -0.002554733838065021, + -0.015155909614094198, + 0.007337783768114582, + -0.0016958649372673703, + -0.003579288757226393, + -0.0013589560854680898, + 0.0025269763572543995, + 0.012096876259066212, + 0.00447616670489574, + 0.001574213659689411 + ], + "4": [ + -0.015474649577819327, + -0.007435491789068523, + -0.14962780387150054, + 0.0015652564371236093, + 0.038648605561990235, + 0.0030858117346523064, + -0.0031410656780368833, + -0.00451824849079568, + -0.003182215599957489, + -0.013839873851308358, + -0.0018576586147573033, + 0.0005357747625475672, + 0.011761435246991952, + 0.0001354722577413672, + -0.004948130560082741, + -0.02034808396536403, + -0.006047515645461685, + 0.0037689516398838096 + ], + "5": [ + -0.016511706702829757, + -0.005653889834639909, + 0.034199673234368774, + -0.004841374561335815, + 0.06676985733658371, + 0.0055472174328561885, + -0.002339091462367892, + -0.007417011743140256, + -0.004699858933020231, + -0.029227628330723572, + -0.004727647679069958, + -5.739095412771686e-05, + -0.008841893701637998, + -0.0050402265336762135, + -0.007477469018998436, + 0.009492560144870432, + -0.002698208273210606, + 0.001739901526121827 + ], + "6": [ + 0.01998897654946294, + 0.002287421119180908, + 0.1336625375190729, + -1.3918148893711985e-05, + 0.04976725344978044, + -0.012117243825648775, + 0.0006085903622452631, + -0.0041714512957865685, + -0.0031566383903831757, + 0.014272369909161744, + -0.005385340073048593, + 0.0022214452540159353, + -0.001958744634860113, + 0.0017733694850172025, + 0.010469779403528136, + 0.010612301346392494, + 0.005332758486995299, + -0.0009604886252799226 + ], + "7": [ + -0.0029980136656270755, + 0.0022824278401606486, + -0.04920309494443549, + 0.0030341564588292126, + 0.2130223951682099, + -0.01666628084292964, + 0.0068167808331864276, + 0.009381646683987603, + -0.0006065719263280659, + 0.04045229210121087, + 8.188915101498699e-05, + -0.00027090795780682135, + 0.012382750582444637, + 0.009020194494612698, + 0.006340825699582828, + 0.02160779281707635, + 0.0017500382301825412, + 0.00016604572359533683 + ], + "8": [ + -0.002598633514788655, + -0.0005707317920156686, + -0.024100304489578946, + 0.0050651355704731844, + -0.08199278192122243, + 0.007372627191382905, + 0.0021819670814611125, + -0.02039925353255501, + -0.003881388226642194, + -0.010885748730999848, + -0.006183598096328545, + -0.0005939208608085435, + 0.008361495246391076, + 0.0006383189366494281, + 0.0014252921424259717, + -0.003875497724120416, + -0.0035034268845453936, + 0.006655785373167392 + ], + "9": [ + -0.022522377972947986, + 0.0033110433183341195, + 0.030887405416661046, + -0.03468135047603532, + -0.037477611363171875, + 0.024512645328787337, + 0.0010906280557058802, + 0.006349436530000566, + -0.003067584894274069, + -0.008474562131661282, + -0.005382116090725169, + 0.007460068892469932, + 0.00828463149355157, + 0.003939514439527605, + 0.011598391577295338, + -0.022121772384988116, + -0.009118737810746937, + -0.02320417415812248 + ], + "10": [ + 0.005768380645276251, + 0.0018694836651851822, + -0.19235538136454833, + 0.0008864790218456556, + 0.1418236290796597, + -0.016030284903252475, + 0.005473047439911256, + 0.004288423271300331, + 0.010469853379032912, + 0.04253889588910001, + -0.002063348686991798, + 0.003971680568877985, + 0.01868557831528424, + 0.0033976254085621364, + 0.01196971395845106, + -0.004155433024500932, + -0.003284532504765063, + 0.014993857269226332 + ], + "11": [ + 0.0021499964786801627, + 0.0017301711805199403, + -0.168478604151011, + 0.00592699017504535, + 0.07016689991335381, + -0.019707181702570575, + 0.008097948862118629, + 0.010895692395215035, + 0.008816482809761952, + 0.028367694648441983, + -0.00041460412679241446, + -0.003319613609808465, + -0.008644610069349929, + -0.0013617784241710348, + 0.004550825060652763, + 0.0048323257590275875, + -0.0075998788490058665, + -0.004012294510670079 + ], + "12": [ + -0.0081327111382397, + -0.001999808247613957, + 0.057968073586795, + -0.004622966686388334, + -0.11186472799305919, + 0.01152471002307939, + -0.001753252141701891, + 0.007502681359009491, + -0.0009293675128535277, + 0.0292883295318258, + 0.01923427854155167, + -0.002040965806166931, + 0.01742245064362167, + 0.0011284650913942359, + 0.0003916364554428673, + 0.013852208265550591, + -0.007328494490400661, + 0.0020014951122170666 + ], + "13": [ + -0.028216207656734396, + -0.007989745760317335, + -0.06566283072568249, + -0.006516370249352153, + 0.19437924000286405, + 0.013450082696953997, + 0.003850613787290122, + -0.016982353853462855, + -0.0006907713534668648, + 0.0050432581249102455, + 0.012596098937620244, + -0.002173682387587276, + -0.01992052261090566, + -0.0031106876404293792, + -0.016396764892051916, + 0.013760426199852063, + -0.019980813003315792, + 0.000649510750701587 + ], + "14": [ + -0.0006895963937810059, + -0.00856197553603908, + -0.08183130369269344, + -0.005190398910516592, + 0.07764503118303544, + 0.016573561617099712, + 0.0009064868129237271, + 0.009764688716479853, + -0.0019856875987766414, + 0.012929355834774912, + -0.007385498706501268, + 0.0003409475761666337, + -0.0016564138736913838, + -0.0009431315165675043, + -0.0011498710887188647, + -0.0005713433589733472, + -0.0035034268845453936, + 0.006117926057297152 + ], + "15": [ + 0.04179912658674907, + 0.0160339182619555, + -0.15646231967531873, + 0.019692574744345492, + -0.14801482613083797, + -0.04211046464372704, + 0.011841206171956607, + -0.007024084754970787, + -0.018265674840307472, + -0.06203662752647869, + -0.009021347337406952, + 0.011573527772367375, + 0.030804430012979903, + 0.004025125380183608, + 0.026368140845046607, + 0.01969643129890447, + 0.01429402636126761, + 0.01799771029478508 + ], + "16": [ + -0.015955237025994894, + -0.007488420546683274, + 0.09363760799998296, + -0.007229407723445398, + 0.12743547263675287, + 0.023927355215735913, + -0.003431141458172476, + -0.004545432118908034, + -0.002255648511509908, + 0.0634850256305569, + 0.002679129310765913, + 0.004386254698037413, + 0.002887514981670827, + -0.0004280547032800144, + -0.002937509625987634, + 0.0316671072176352, + -0.00793045403479657, + 0.0034862444811762316 + ], + "17": [ + 0.024896027336097665, + 0.03800534460690165, + -0.3332249801475088, + 0.027709963820542582, + -0.6991009276975156, + -0.08583772078786346, + 0.020451053469608634, + -0.005698676947916958, + 0.0027404953236594884, + -0.16540234228120046, + -0.015401608355464403, + 0.0014111755990285644, + 0.02014598961923488, + 0.05213000701274839, + 0.005918481922543114, + -0.05489255894264814, + 0.01433646506755155, + 0.03347288544814013 + ], + "18": [ + -0.00013845178562398573, + -0.0017756100196043025, + 0.025888522697911955, + 0.011837920947521032, + -0.12305293989266039, + -0.00955442647419341, + -0.00628355329617247, + -0.008591673976116489, + 0.010241338747507657, + -0.006238734557729157, + -0.0076021503186354065, + 0.0013022460578058911, + 0.002703041974856009, + 0.000903148384942448, + -0.0023172217297313196, + 0.01055264300368845, + -0.002248134624988804, + 0.005582571741567359 + ], + "19": [ + -0.012762858353626453, + 0.001766622117367835, + 0.025976578897564717, + 0.002935658789735251, + -0.01653304680707907, + -0.04808650849604665, + 0.007551923864216336, + -0.020274044093976897, + -0.000320741941961101, + -0.0007756264585284899, + -0.008036098339368369, + 0.00343213008566412, + 0.007700466971971313, + 0.0017286824555539043, + 0.014305832501472746, + -0.008060759919973369, + 0.00720229517960894, + 0.0095447662494285 + ] + } + } +} \ No newline at end of file diff --git a/data/mi1v2_patching/gpt2-medium.json b/data/mi1v2_patching/gpt2-medium.json new file mode 100644 index 0000000000000000000000000000000000000000..0b42cb1fb67ca231c829b9d5d1faa487d66d7ed7 --- /dev/null +++ b/data/mi1v2_patching/gpt2-medium.json @@ -0,0 +1,1104 @@ +{ + "model": "gpt2-medium", + "N": 24, + "L_crit": 23, + "N_sem": 1, + "n_heads": 16, + "D_LONG": 200, + "N_PAIRS": 20, + "n_pairs_with_signal": 19, + "mean_gap": 0.3570153838709781, + "head_summary": { + "21": { + "per_head": { + "0": -0.005820556273283774, + "1": 0.05833076321918012, + "2": 0.003121889086426487, + "3": -0.0003383132783011527, + "4": 0.010246272633963894, + "5": -0.0013764902795908745, + "6": -0.01039606991495497, + "7": -0.007577217818330506, + "8": 0.0035625385765766407, + "9": -0.003921379441788467, + "10": 0.0036250355000705128, + "11": 0.006907835621221703, + "12": -0.002701671778420181, + "13": 0.010754335619250035, + "14": -0.0080009794256831, + "15": 0.0006292902043543336 + }, + "top": [ + 1, + 0.05833076321918012 + ], + "n_critical_005": 1 + }, + "22": { + "per_head": { + "0": 0.0017449168862459642, + "1": 0.016615361132706844, + "2": 0.015919488331256167, + "3": -0.013828745483345062, + "4": -0.02763331357924657, + "5": 0.007384635841517145, + "6": -0.015565991382469767, + "7": 0.004453180588333858, + "8": 0.003251307245891224, + "9": 0.0948391829067483, + "10": 0.007719265390701722, + "11": 0.01906678120623878, + "12": 0.007313881907439051, + "13": 0.002419787536528843, + "14": 0.016818915807180593, + "15": -0.004624667446061916 + }, + "top": [ + 9, + 0.0948391829067483 + ], + "n_critical_005": 1 + }, + "23": { + "per_head": { + "0": 0.001634201293533084, + "1": 0.020112480872571406, + "2": -0.005019999936281006, + "3": 0.003469793332034483, + "4": -0.005407218808121697, + "5": 0.028455352354652897, + "6": -0.010235767413762406, + "7": 0.0015458081442068547, + "8": 0.015585066616018184, + "9": 0.006741690207014342, + "10": 0.004046567314109137, + "11": 0.01796507144235075, + "12": -0.004073175434368562, + "13": -0.005205982519008448, + "14": 0.013329400562016772, + "15": -0.02778463049882472 + }, + "top": [ + 5, + 0.028455352354652897 + ], + "n_critical_005": 0 + } + }, + "raw_patching": { + "21": { + "0": [ + -0.010874544702724791, + 0.001600684427383398, + 0.017509894156232414, + 7.789573352600926e-05, + -0.07536183206760072, + 0.006738034708332692, + 0.008512377484872635, + -0.03223701585072198, + -0.022339022976358796, + -0.006059284921212541, + 0.02470843507156721, + 0.004973300632269372, + 0.0052183489754879814, + 0.005690593244197678, + -0.012794052381069253, + 0.0006026351990695523, + 0.00627862454840133, + -0.03642397992061106, + 0.0035883394465671343 + ], + "1": [ + 0.046763683269562505, + 0.09279037898270484, + -0.041868590223843566, + 0.03908021002455647, + 0.22885912432536545, + 0.7100007285508719, + 0.015371244476002311, + 0.3120161641341547, + -0.06937779845254333, + -0.011388362378098071, + -0.10105336166650174, + -0.03843239049678143, + 0.05524841770491138, + -0.19549777555788336, + -0.0024853873001817193, + 0.05623717993063901, + 0.002970830357984243, + 0.03328035485687762, + -0.024230149373374886 + ], + "2": [ + 0.0031742362759711144, + -0.0013583839045510817, + 0.006213041760471662, + -0.002347999967712565, + 0.023225119655633814, + 0.04115238955042443, + -0.02758989605163365, + 0.015840811854544504, + -0.010840421885608256, + -0.003217735427359617, + 0.00592687182577944, + -0.006256268041753356, + 2.0280747394749794e-05, + 0.015815616119570868, + 0.0012407749259785218, + -0.0028285884203364388, + 0.003947620884247545, + 0.0027065560028277473, + -0.005508133261786175 + ], + "3": [ + -0.00018105473904204172, + -0.0032549751651279276, + 0.002782964374863479, + -0.0007014590289459508, + 0.002139120530818607, + -0.008213947684749024, + 0.0007062260081476191, + 0.003734976458899362, + -0.00022991191412504342, + -7.282589454095266e-05, + 0.002429465745075149, + -0.0006816407905926138, + -8.816491575773175e-05, + 0.0005205335519715142, + -0.00035432438607840264, + 0.0006394775256502236, + -0.0013779104228471663, + -0.00162907559064848, + -0.002595425950692521 + ], + "4": [ + 0.007239522634265338, + -0.002423005228323161, + -0.017313111252380223, + -0.004571207790898767, + 0.1251345351820547, + 0.07911287130385249, + -0.017608568469813967, + 0.006668988929479709, + -0.0416102997260099, + 0.00264109561021679, + 0.001046266269776889, + 0.016948349566698523, + -0.0021145495929498154, + 0.019547110359075755, + -0.004961820554505862, + 0.011930146854194119, + 0.002103278535545654, + 0.005278719112596258, + 0.007630858302439444 + ], + "5": [ + -0.0030462237599872158, + -4.824567932501869e-05, + -0.0020480559147077982, + -0.00213974810869405, + -0.021988231438815044, + 0.00452613312767675, + 0.006855100452419556, + 0.005798065519721066, + -0.0011578243779957252, + -0.0005251258482618526, + 0.0030767321662083097, + 0.0003130193358612607, + 0.0011345951459173912, + -0.0066155082332379714, + -0.008147542155690923, + -0.0014349710261063512, + -0.005018622567777662, + -0.0007198784532564302, + 0.005033016503825087 + ], + "6": [ + 0.0012098960712089303, + -0.0007719308692002991, + 0.020258043094260506, + -0.008741808952235202, + -0.03468106052093146, + -0.08518969699875066, + -0.003948274202883955, + -0.011286310744495206, + 0.016476269166170187, + -0.002788087006054465, + -0.019621925889968268, + -0.02848372989450676, + 0.014653403346801024, + -0.010492407712880356, + 0.014073201789294533, + 0.0008481253955713723, + 0.010178272880175587, + -0.030872969120338264, + -0.038344338215381656 + ], + "7": [ + 0.004513330164237868, + 0.07693148812546759, + -0.032052907608231804, + 0.0014386865069599668, + -0.12316140397903427, + 0.005285736339539022, + -0.006530236488671651, + -0.04707126868251264, + -0.0007205409334833877, + 0.0011883529748003612, + 0.037896118957851055, + 0.002215847403859977, + 0.0011974091274316858, + -0.04259168995577829, + -0.0011755383061590324, + 0.006209811739382947, + -0.012550891288643548, + 0.012859646914989866, + -0.027849089560285277 + ], + "8": [ + -0.0001700907041082356, + 0.003727782822513111, + 0.0015742632308175242, + 3.4178740220595895e-05, + 0.003721373639346513, + -0.002703872572794719, + -0.0033974179165288127, + 0.045680646713488184, + -0.0032743664109703896, + 0.0012481481490806416, + 0.009403589025442945, + 0.003540001568062547, + -0.0005996904333808654, + 0.0067402642085038714, + 0.0012580434429895631, + 0.0021943740841977397, + -0.0003947620884247545, + -0.0006357368158628214, + -0.0002584957276364565 + ], + "9": [ + -0.0020644981455088455, + -0.007891921011365835, + 0.04404379816873278, + -0.0005079119767665297, + 0.003965003136598695, + -0.036409789186206624, + 0.01600684788333517, + -0.01116495256444687, + -0.020235253827502186, + 0.00036486016729881965, + -0.009607522281416406, + -0.002683317069915939, + 0.0014979585367399918, + -0.0034088494758861805, + -0.005445339030815018, + -0.008303057396537824, + -0.0007542672406944431, + -0.01869814164302416, + -0.013209856436599476 + ], + "10": [ + 0.0015426693477133702, + -0.0019062403964418496, + 0.01990080643803653, + -0.004043027026327001, + 0.033508426237453925, + 0.11311790628048476, + 0.012963484605557695, + -0.005423996776513255, + -0.005929623615277264, + -0.005799401210593154, + -0.0027476804391786663, + -0.00987246510604529, + -0.0064194199045326085, + -0.03112790640789655, + 0.0006651576922771458, + -0.002875581183832193, + -0.004785695596130259, + -0.012523080365415431, + -0.01936865807200016 + ], + "11": [ + 0.0028776146822213867, + 0.0006432757243335825, + 0.004303490735782905, + -0.006565894967261219, + 0.016068837829753792, + 0.10589970788382462, + 0.0032881882939353145, + 0.004544506906986494, + 0.0020038401142859178, + -0.0011504786383417722, + -0.011481343792824733, + -0.004909461162455563, + 0.004504579338011649, + 0.009822166990589763, + 0.0007271964385760718, + 0.002298510048104331, + 0.006990121079661758, + -0.0033574850587755256, + -0.005258495643196451 + ], + "12": [ + 0.0014881454983128209, + 0.0038596543460014954, + 0.0046576999627168485, + 0.001393379804807084, + -0.0855032445466256, + 0.05412468067113971, + 0.003868235255293892, + 0.0020059793290342523, + -0.0003576407553056231, + 0.0009195791466199557, + -0.02786890794675013, + -0.017829746117676286, + 0.006410406239023831, + -0.009632021676563804, + 0.005738264245298607, + 0.003718819332000007, + 0.006239321734326829, + -0.010466284784682773, + 0.00590191647304545 + ], + "13": [ + -0.005215324725269942, + -0.03115598758188985, + 0.01886693933549002, + 0.005021492821944525, + 0.20676380299764835, + -0.16809271282604027, + 0.04557794329649227, + -0.08724082627851171, + 0.010906540344572321, + -0.0010033654600717541, + 0.05009172173898863, + 0.03422207449567389, + 0.02346200796525402, + 0.09023040440058948, + -0.006391909592901725, + 0.006127856359846352, + -0.0032499959344251747, + 0.014269019341332812, + 0.0011426960670284478 + ], + "14": [ + -0.004951002585784669, + 0.007854396594113043, + 0.004467728774767234, + 0.003676996564197131, + -0.09425516571714629, + 0.036417660722080844, + -0.02365857127294524, + -0.008826309047750709, + 0.02745644143095249, + -0.013917844389072748, + -0.029858981265911445, + -0.04745331944901994, + 0.009683775203945878, + 0.0555921229645214, + -0.015319093312905955, + -0.01178503320051923, + -0.011802981856108216, + -0.010969965975191736, + -0.024369463270200765 + ], + "15": [ + -0.001963154903688259, + 0.0014216393507772173, + -0.00490670602182212, + -0.0013635727639170294, + -0.13634149205565504, + 0.06476306140514863, + 0.010104681724576134, + 0.016697457831356285, + 0.039054971556273715, + 0.003966940951132094, + 0.025332057057225188, + -0.02749009943692085, + -0.01415398994220531, + 0.0036901957097616933, + 0.012602819544539573, + 0.001078577907754551, + 0.0011894881083135355, + 0.01183942956159236, + 0.006434208298489992 + ] + }, + "22": { + "0": [ + 0.004967004150282657, + 0.00603607054666345, + -0.027852349468310042, + 0.01580846220644934, + 0.008652863023286281, + 0.027613347846765286, + 3.5782117746146035e-05, + 0.0008737788963480159, + 0.0021075258794795645, + 0.004422772595390932, + -0.0182968523185465, + -0.0007084121811596953, + 0.0012238867698637203, + -0.004733843971813672, + -0.0016411486907530345, + 0.0008274485796332405, + 0.002727499700258296, + -0.002659810648720187, + 0.013749395805809526 + ], + "1": [ + 0.0023252643817723425, + 0.009943970571989964, + -0.033187436734671935, + -0.017498322711309498, + 0.21971632506035224, + -0.03693324632184228, + 0.010992643225487073, + 0.06033357614685368, + -0.03831865235417391, + -0.005517596662152077, + 0.12379635303919428, + -0.0482296897754653, + 0.005390735328343355, + 0.07929059594882452, + -0.04287325071548671, + -0.008715465888976563, + -0.006868166759518825, + 0.0433528100332042, + -0.0013085842909945228 + ], + "2": [ + 0.004094029909336904, + -0.012833350700454972, + 0.01706486205059746, + 0.0018090886684203782, + 0.06500891705799426, + 0.08059272004820592, + -0.026053148257904432, + 0.04299363383288857, + 0.02717769201872899, + 0.005007815400365408, + 0.04217773214848228, + 0.01882852491960202, + 0.0063512540591224775, + 0.026136807011224228, + -0.009551408631218167, + 0.032403578111916354, + 0.013880829159017428, + -0.024391725773325015, + -0.008227572739132947 + ], + "3": [ + -0.0011900422782206866, + 0.0021217377640936, + 0.0016976309743863974, + -0.00713302359859599, + -0.059670486096226674, + -0.08456784566468724, + -0.014404185662178838, + -0.006167851033044818, + -0.07683115200849246, + -0.0013748018787170812, + -0.002498428681877769, + 0.014129116206211248, + 0.001840759503120693, + -0.01320434449935013, + -0.0027143550442540448, + 0.0005631612777331187, + -0.009218821830709859, + -0.013168166252099765, + 0.00904493461935414 + ], + "4": [ + -0.002597290978238127, + 0.019014158285093476, + -0.023896256243558504, + 0.002220425832703131, + -0.557424289712992, + 0.02298882052066078, + 0.05124846732457889, + -0.01511694600413855, + 0.01246783759147376, + 0.003128834251749424, + -0.04135313246128524, + 0.02500962713399395, + -0.01099526353547303, + -0.024742981632391414, + 0.03642250024980663, + -0.001487978863329562, + -0.011572944797260409, + 0.005227299223077942, + -0.013573844190156108 + ], + "5": [ + 0.002831387940338312, + 0.013573117783438591, + 0.0012525988687514436, + -0.0014219945640615364, + -0.16818199327345942, + 0.20469535464116528, + 0.02573958391028689, + 0.04747674777820355, + -0.019579328660733683, + 0.00622515259406334, + 0.0361070233402288, + -0.005597279966255964, + -0.0014895082253255127, + -0.032642185831813685, + 0.019200032617461456, + 0.002601519387124954, + 0.00660402872963579, + -0.01672782496739049, + 0.019641648887167632 + ], + "6": [ + -0.004676605387171035, + -0.0142271147698444, + -0.0028813558267895742, + -0.008311395281782815, + -0.20357520155558956, + -0.05574621706122912, + -0.025230159549743073, + 0.03456280967776596, + -0.010254371908426773, + -0.007204892262260604, + 0.024114368630253215, + 0.014335049979804183, + 0.0011168494919469852, + -0.024065857821810286, + -0.03947774861135271, + -0.00271091854054307, + 0.0008259370781244131, + -0.004849830488659391, + 0.03250281794038211 + ], + "7": [ + -0.015798285363809186, + -0.026639119870860876, + -0.034106261524197164, + -0.0061434298410461795, + -0.014063579660062758, + 0.06382241286817929, + 0.02324990182289715, + 0.055407861780185946, + -0.04403339097781796, + -0.0031133678828418974, + 0.021586384404031468, + 0.07712631688602589, + -0.012653496312040977, + -0.015549756834349051, + 0.010042602003976674, + -0.01095946433142582, + -0.003218206893629624, + 0.05215612884551051, + -0.03250281794038211 + ], + "8": [ + -0.0046727531586807786, + -0.025960463981688945, + 0.014536958593419321, + -0.007632192176701438, + 0.005643637145138452, + 0.17294945046043428, + -0.049845431655058954, + 0.01778682596520193, + 0.04493951440407548, + -0.015326075545467908, + -0.010752307032339894, + -0.014046742696774075, + 0.007034039221412387, + -0.042300019089466984, + -0.001980123283932734, + -0.0015278287267739617, + -0.04067205475894809, + -0.007891784407208884, + 0.02149218839529404 + ], + "9": [ + 0.06135088726108883, + 0.004504002196542301, + 0.08749724762590899, + 0.016954642285474903, + 0.20536360270025394, + 1.1656052249507507, + 0.10318244469439973, + -0.09712794859421434, + 0.013067411798897893, + 0.012307454394988655, + -0.13127686131265562, + 0.11085620966281266, + 0.05470027417115884, + 0.02012270861737153, + 0.04172777242042098, + 0.05263415076789254, + 0.057354943368453384, + 0.020675470121773964, + 0.002444838096898074 + ], + "10": [ + -0.0019270032209335471, + -0.01007262571685668, + 0.007999225041591545, + -0.007988286958534622, + 0.021155607552601544, + -0.03025818390050335, + 0.0066347579378774986, + -0.010011335982340339, + -0.0037687521609516923, + -0.002072736998473268, + 0.15194308310277782, + 0.016909222149715866, + 0.005803955556511382, + -0.00980495926986343, + -0.014756267573286833, + -0.007522225638292372, + 0.00670228576482204, + 0.028078765578558842, + -0.00037848284108764655 + ], + "11": [ + 0.005833162910053341, + 0.03272558034926379, + 0.027860674898857636, + -0.0037441617630027207, + 0.05544043548459539, + 0.17281957011850965, + -0.0013135803751545714, + 0.033256424563127995, + -0.023516382353593903, + -0.006521205687088517, + -0.0003300369118411091, + -0.005249251888883905, + 0.022595851045269913, + 0.010063935466794714, + -0.009010328431538874, + 0.002553774739413268, + 0.03013254277882306, + -0.0006871567053811379, + 0.019358994680312814 + ], + "12": [ + -0.0002456536475709535, + -0.02051513497520517, + -0.032654609179626, + -0.0027970927171227203, + 0.14725930622921984, + 0.09964970839969359, + 0.0018559619494119429, + -0.013015307874360315, + -0.04117451897374675, + 0.0009522168384878074, + -0.02037460708592829, + 0.009409114115461185, + -0.0046206302814371615, + -0.016923793334346585, + 0.001624519748446106, + -0.011882026264374465, + 0.01173420193147784, + 0.018873436720927513, + 0.011808664641934573 + ], + "13": [ + -0.0004107068221150079, + 0.006583927038554217, + -0.016427588185045368, + 0.0004884380433850275, + 0.05106045902850122, + -0.05870985031787311, + -0.0075490852097592825, + 0.008136709036417226, + -0.014188420216788627, + 0.0025124933616628665, + 0.009948396226183641, + 0.025906468717991182, + -0.00044251464107155454, + 0.0370146676683744, + 0.005030894622550027, + 0.013124890873310174, + 0.00953786820378521, + -0.012704218612582228, + -0.012936865621432004 + ], + "14": [ + -0.00010519547031084257, + 0.0010667655761865244, + 0.04067275565428026, + -0.004180934268844987, + -0.05419551552610897, + 0.15240474182871894, + -0.019172623669191563, + 0.008456523534426958, + 0.03876946002892889, + -0.006491612556029233, + 0.050243440199954394, + -0.059098874345700395, + 0.003082673604001969, + 0.015401770436102604, + 0.008838922410836687, + 0.006237631455372434, + 0.014297554584719109, + 0.01814187192914419, + 0.10519004492994433 + ], + "15": [ + 0.002884430163396455, + 0.011201574613062117, + 0.034234170411701084, + 0.007739100096693766, + -0.006826980417506193, + -0.024004248648435218, + -0.015982365381719384, + -0.016840232160824913, + -0.011100387644716965, + 4.481593510212471e-05, + -0.0333691947491648, + 0.0008875745641855485, + -0.01119356417666614, + 0.019692515599213253, + -0.010334248069052039, + -0.0037755865903028782, + -0.006054945297712632, + -0.005753184456787996, + -0.019317925265641602 + ] + }, + "23": { + "0": [ + 0.008063899531180165, + 0.010178766211371721, + -0.06541215095511314, + -0.00019275219775568618, + 0.10157475962360195, + -0.05380981923617089, + -0.009115023678491936, + 0.017755415612718834, + -0.015845138421513204, + 0.000633268648182197, + 0.06391583501347951, + -0.01733756439878917, + -0.0029567639639262303, + -0.023913569493382258, + 0.0004579354881446503, + 0.006511317309971706, + 0.006530624944526063, + 0.010620544453237723, + -0.00660975991414341 + ], + "1": [ + -0.0024437944891648414, + 0.044230566678969914, + 0.05463753011188321, + 0.008090028324772675, + -0.03982405243545279, + 0.2289987216528209, + 0.011378713443274438, + 0.050633488202754956, + -0.012099677990423854, + -0.031709161712454546, + -0.07921575513554645, + 0.05520260734932207, + 0.013041647283012716, + -0.0499522924964664, + -0.006690590979722328, + 0.014980541118230517, + 0.0006126615135142602, + 0.1228175747483515, + -0.0005516186088192295 + ], + "2": [ + -0.0009200899586342709, + -0.009440071254595324, + 0.02052067258517093, + -0.010672510347487002, + 0.09800331182871831, + -0.07729848228484466, + -0.0029200091350210225, + -0.015635216820109675, + 0.0012104186067171405, + -0.0058723488875664495, + -0.015271349762534425, + -0.02245295933483767, + -0.00397474481232381, + -0.017526063559768174, + -0.014911044651682092, + 0.01100495332648971, + -0.020214362050611074, + 0.007299287043895556, + -0.01630938932031512 + ], + "3": [ + 0.0067603646751311615, + 0.0075584897609195945, + -0.00136007260854764, + 0.004591476578704005, + 0.000543481186177944, + 0.014814230515282857, + 0.002259923226072381, + 0.027031463798295727, + 0.01663330050620984, + -0.0004082147132128316, + -0.020531251470951382, + 0.006750509098376399, + 0.0018182253393487489, + 0.0018747811731337678, + -0.006216666123974862, + 0.0003045131074524874, + -0.0022385264545667265, + -0.0015566202917817612, + 0.007296666006585458 + ], + "4": [ + 0.001294941423263048, + -0.04257413168881094, + 0.02134716078135013, + 0.007069832671909075, + 0.19102158932904584, + -0.1277747060782832, + 0.0019021020486109208, + -0.03220275001164951, + -0.07244930140987398, + 0.0011611954923879324, + 0.1082284626484086, + -0.05354484047189895, + -0.008746917345127296, + 0.0002004699464617567, + -0.022655015169077936, + -0.021445241606814436, + -0.0024737653564538054, + -0.023106228535367104, + -0.02799001402239238 + ], + "5": [ + 0.03183600154455122, + 0.1108514170695774, + 0.009581056845634154, + 0.06656349400682238, + -0.06346146816335953, + 0.47198122678619436, + -0.07683362314775083, + 0.05039505507254235, + 0.012461826822477026, + -0.009489713366658737, + -0.13569541519208062, + 0.048056705405647236, + 0.029953818870904, + -0.15216529322483652, + 0.032586970749243126, + 0.03959911005838625, + 0.024607607488556255, + 0.011207198647287605, + 0.03861571846526791 + ], + "6": [ + 0.003263726507052451, + 0.02185314848181902, + -0.039163582153198456, + 0.009683711444360926, + -0.25777607157614363, + 0.08526054082161864, + 0.002633752193051854, + -0.0038649010987158155, + 0.005508118439381351, + -0.0022873176442611584, + -0.04161617680594666, + 0.008332080479570137, + 0.0027928279224853363, + 0.021595689511545467, + 0.006374001501186571, + -0.0017518902231217178, + -0.0036834828543645098, + -0.005070702286817615, + -0.0065630535209879135 + ], + "7": [ + -0.001583265909495301, + 0.0004813846670429643, + 0.014432512282913158, + -0.005131977586843661, + -0.030895432948243713, + -0.04802817613655598, + 0.0275757715314707, + 0.029482899035272105, + -0.010249112485554632, + 0.0009447881101149008, + 0.04741694497585845, + 0.011270755428741315, + 0.0018190703704901968, + 0.020401473693138092, + -0.00497333289917989, + -0.008124860837361924, + 0.00266623354890687, + -0.012648124187653155, + -0.005487195913130263 + ], + "8": [ + 0.012918003754171467, + 0.02827840084170429, + 0.0362300031711558, + 0.018367098596451623, + 0.06582815470809501, + 0.2574582596060649, + -0.04529074471984557, + 0.05188561907219485, + -0.004112117339890074, + -0.015209590648932087, + -0.08206195405587172, + -0.03743258202598774, + 0.027401824823731314, + -0.1155610296957851, + 0.007176028180143822, + 0.04115475850114969, + 0.028474310814415125, + 0.022281173035368664, + -0.0016693509139887048 + ], + "9": [ + 0.0037654051865912016, + -0.01780372779713912, + -0.02735358049277699, + -0.001434712234841293, + 0.10703902120482944, + -0.061634125895146004, + -0.001175160077557638, + 0.019413025577849626, + 0.011282213406868144, + -0.0029272843262222055, + 0.03425881663391859, + 0.03799272189016052, + -0.005244544940872867, + -0.018633380388507575, + 0.03425817945108946, + -0.002163546831344525, + -0.0001849544191741163, + -0.0016851700155775524, + 0.020322918001125397 + ], + "10": [ + 0.0021845098792437506, + 0.013189296601252887, + 0.035261225798345015, + 0.008466789321622965, + -0.04188285554959485, + 0.05332178401196923, + -0.025711334869960984, + 0.00829376079883272, + 0.007925950268317265, + -0.001989316032318498, + -0.03856998536889379, + -0.00024815019717948627, + 0.0001366133678674118, + -0.008793145291155166, + -0.014772896515593761, + 0.004513560948240202, + 0.01793422085173017, + 0.014025943499973497, + 0.043598807445375046 + ], + "11": [ + 0.011881754290292548, + 0.008469797037058836, + -0.006352303507813212, + 0.013644868465043244, + 0.1604420715530632, + 0.16192536446858857, + -0.040682384608012975, + 0.013212336449027024, + 0.054718284215635744, + 0.005274153579898959, + -0.01081240330583932, + -0.043246092454516286, + 0.025284458460310008, + -0.05443232258756669, + -0.00694066468903037, + 0.009195543960849189, + -0.006068816879150691, + 0.02149351381865627, + 0.024329199138170166 + ], + "12": [ + 0.0029869587062909667, + -0.011073991594402623, + 0.027581394546852028, + 0.0009367359383714481, + -0.05543240374292773, + -0.01898220876068258, + -0.02907485393809871, + 0.05551065929740336, + -0.04672321010385605, + -0.012780640035856335, + -0.0023506509601578696, + 0.030978617561585164, + -0.011310741828280249, + -0.021162054949241924, + 0.0032669475886073656, + 0.0021620430629126605, + 0.0021385354717007197, + 0.010982820947571316, + -0.005044290460793656 + ], + "13": [ + 0.0014819226676747147, + 0.014544464127182301, + -0.028408639600353735, + -0.0037028293329685116, + -0.01983304742466113, + -0.06576274646117462, + -0.012077406374001816, + 0.010992195625789828, + -0.018657426965859734, + 0.0035210954663253582, + -0.0062805531730957325, + 0.02948147902756453, + 0.008215392757156562, + -0.011383767646504304, + 0.001249089397131986, + -0.0036267135155483286, + -0.000367018925548637, + 0.0018417669518378797, + -0.00014092446210710243 + ], + "14": [ + 0.00013067944340022978, + -0.02429223560325052, + 0.01987280271710372, + 0.0021174921848294763, + 0.05061335874233513, + 0.08591781406711604, + 0.012446527167593638, + 0.05365459301431117, + 0.03362574445997449, + 0.0020441181268727265, + 0.032893350485226715, + 0.019252748493203464, + 0.003989110341728425, + -0.01102584705539662, + -0.006957293631337299, + -0.0033564111399207504, + 0.007728204808678466, + -0.02196564189514263, + -0.0034305040490071794 + ], + "15": [ + -0.012087107701350052, + -0.07383625976521582, + -0.01802228656357062, + 0.014800984224365493, + 0.13793177690585062, + -0.5771803679772134, + 0.03169071507227749, + -0.11898384294927222, + -0.006611094550281886, + -0.012307576177420999, + 0.20414112996868078, + -0.06350482743172123, + 0.02196714787603267, + 0.0376694214420113, + -0.011226454781289173, + -0.015167760287997602, + -0.008988206789302133, + -0.015292742596288385, + -0.04290062739596443 + ] + } + } +} \ No newline at end of file diff --git a/data/mi1v2_patching/gpt2.json b/data/mi1v2_patching/gpt2.json new file mode 100644 index 0000000000000000000000000000000000000000..75629e6fca124978c64a217529ecd6ebea5f86a3 --- /dev/null +++ b/data/mi1v2_patching/gpt2.json @@ -0,0 +1,876 @@ +{ + "model": "gpt2", + "N": 12, + "L_crit": 11, + "N_sem": 1, + "n_heads": 12, + "D_LONG": 200, + "N_PAIRS": 20, + "n_pairs_with_signal": 20, + "mean_gap": 0.016157364845275878, + "head_summary": { + "9": { + "per_head": { + "0": -0.029539912804184464, + "1": 0.14914260824691558, + "2": -0.0030281481727417988, + "3": 0.03836693054617849, + "4": 0.07209714848774532, + "5": -0.03794978669095349, + "6": -0.013438869047988388, + "7": 0.00828759921331795, + "8": 0.004194568651683403, + "9": 0.024948476081012708, + "10": -0.02418548880829011, + "11": -0.04995625433003219 + }, + "top": [ + 1, + 0.14914260824691558 + ], + "n_critical_005": 2 + }, + "10": { + "per_head": { + "0": 0.0405114151037414, + "1": -0.011176030939966237, + "2": 0.03750057102575832, + "3": 0.008003285455097776, + "4": 0.06426280744368866, + "5": -0.05120816276939881, + "6": -0.029381750547022634, + "7": -0.022356234048566962, + "8": -0.02910665971919213, + "9": 0.05766888572828584, + "10": -0.004633291281753492, + "11": -0.02414702905578479 + }, + "top": [ + 4, + 0.06426280744368866 + ], + "n_critical_005": 2 + }, + "11": { + "per_head": { + "0": 0.04038939331495187, + "1": 0.007741758948758515, + "2": 0.021736341998224173, + "3": -0.046315649800551806, + "4": 0.04636696775655441, + "5": -0.06881682928988093, + "6": 0.02623944519033831, + "7": -0.04803592995249255, + "8": -0.0006659990516335284, + "9": -0.017599218783733068, + "10": 0.06405375966494742, + "11": -0.007814258882257459 + }, + "top": [ + 10, + 0.06405375966494742 + ], + "n_critical_005": 1 + } + }, + "raw_patching": { + "9": { + "0": [ + -0.041183945930712014, + 0.013613466089057726, + 0.010989874538079097, + 0.02169127084895274, + 0.007517567144258878, + -0.017941308596066893, + 0.08015706643981725, + 0.030843757470218457, + -0.27943377966841537, + -0.022520547350632244, + -0.0052541168105498075, + -0.20415612985979412, + -0.017759500125866873, + -0.03116151495823861, + 0.0010053351841606572, + -0.006613415463345065, + 0.023043997710898004, + 0.06270471669945299, + -0.16554921060320574, + -0.050791838841758474 + ], + "1": [ + -0.04743247144106997, + 0.0046219812232773504, + 0.028336252096778508, + -0.1813166476144918, + -0.00384222885876089, + 0.20684502345850328, + 0.03536891135377612, + -0.4434289343339926, + 0.4208577088494376, + -0.03672317534957458, + 0.03831569324776687, + 1.4937081010386608, + 0.049779549195345336, + -0.01581411993360294, + 0.01866527478741842, + -0.003989265947517846, + 0.022782611469899364, + 0.07771532659005741, + 1.2169034921464321, + 0.10149908295996933 + ], + "2": [ + 0.022293073835844054, + -0.0039935230965202064, + 0.004863496571626934, + 0.012619642705803359, + -0.0009559791690396191, + -0.01859673876884089, + 0.02011671372508627, + -0.21230043610576338, + -0.12515725945856268, + -0.07827272710426772, + -0.0017591966779715539, + 0.02375074821846701, + 0.022919466692932332, + 0.0915631182675627, + 0.0018830897611943275, + 0.02371777564545851, + -0.07383227785922314, + 0.0946102483542845, + 0.10490403510852407, + 0.031063765898569146 + ], + "3": [ + 0.04768551427689113, + 0.03977108145448192, + 0.010608203716531917, + -0.022511077244111423, + 0.058776279550191234, + -0.09138541523029926, + 0.08048080933550399, + 0.40007750940399894, + -0.10994940840976705, + -0.1455788438940997, + -0.0021001789732417067, + 0.21353202511839367, + -0.014578114720247707, + 0.16831826904783576, + 0.03949106916534731, + -0.009848791288903678, + -0.1909519842723997, + 0.011524394101604806, + 0.0599646077129212, + 0.2240126620729382 + ], + "4": [ + 0.048655239725134795, + 0.015892799295057516, + 0.017041181221616648, + -0.02309569328000327, + 0.025466003210433286, + 0.16893979730357211, + 0.42215236331441097, + 0.3110421329423044, + 0.31252273683317777, + -0.11712210146281393, + -0.0038147495461035253, + 0.29712045985353597, + 0.011275176081487258, + -0.19243282209821813, + 0.04928554835685622, + 0.02711011491943563, + 0.25489203760621, + -0.13536237349183688, + 0.16392902020662758, + -0.21155390123597803 + ], + "5": [ + 0.01635554613338242, + -0.010616565021909854, + -0.00872966764172939, + 0.007989752490521935, + -0.009756647653372134, + -0.03769743051497023, + 0.0049594062554770115, + 0.058683591103498765, + -0.027455902042827985, + -0.0062508343958825145, + -0.0036364609713966756, + -0.030869785049067434, + -0.0078061359042457925, + -0.2121748313561061, + -0.0044592901661985405, + -0.0064601100613939375, + -0.36244780816570416, + -0.008267448198369172, + -0.16078523613088241, + 0.05043012347210662 + ], + "6": [ + -0.1033402453477241, + -0.0447205800348628, + 0.0016585842318704662, + -0.039560698359819395, + 0.019527445629664315, + 0.10162080700987068, + -0.0445258117050536, + -0.10626018828968674, + -0.08687904245752696, + 0.23056196948497834, + -0.012157564821437044, + 0.08722936939964883, + 0.008896932980707336, + -0.06050416594578699, + -0.016664346938186806, + -0.0007299465450818333, + -0.04634315818086611, + -0.01806575052980822, + 0.1620849010560508, + -0.3006058915967179 + ], + "7": [ + 0.011289791813685295, + 0.02060623524573609, + 0.004586364265540287, + -0.053640201224214756, + -0.002407635826249106, + -0.07250320020842792, + 0.2943771823383254, + 0.31836383555064107, + -0.666856580697339, + 0.022156328506942935, + -0.01750114968793841, + 0.05478987906588359, + 0.013956347658577388, + 0.32500671407751086, + 0.006989097623156576, + 0.04308945953799015, + -0.06832138461150179, + -0.09589511500575344, + 0.13146374163671184, + -0.1037977257929182 + ], + "8": [ + 0.033946920826226884, + 0.017816475227830312, + 0.001220083747556151, + 0.007417735938162238, + -0.03538413101948125, + -0.03441445360512003, + -0.11256205576584065, + 0.139274410960005, + 0.17300328348432326, + -0.022505292111105992, + -0.02086909856861436, + -0.08146835560580978, + 0.009428967027336187, + 0.026682513745618124, + -0.005429366709844103, + 0.003660374315132455, + -0.02675786055175369, + 0.0012514314597950303, + -0.0299230285980492, + 0.03950281883730147 + ], + "9": [ + 0.009148722915753365, + -0.09079656602967198, + -0.008818770940142058, + 0.009101026938057734, + 0.02427737608514937, + -0.009758627016857296, + 0.09097454457500515, + 0.016559202499961533, + -0.22601760063994797, + 0.17898972536153643, + 0.017705177866197068, + 0.5255484019267175, + 0.032153693985497674, + 0.028901242172391577, + 0.005993504958085616, + 0.007846177122852288, + -0.04941600238308227, + 0.0044630010314732115, + -0.039784675293752576, + -0.028100033514970098 + ], + "10": [ + 0.12410118562603165, + 0.007111582135866283, + -0.003696032882188501, + -0.020086936612525395, + -0.016052292794888165, + -0.014285464743483038, + 0.19329404493420504, + -0.00398208015378313, + -0.7187806291196861, + -0.025491505248370183, + -0.00783460649912616, + -0.17395395805421848, + 0.029654534055938465, + 0.03457853849204068, + 0.029103224168327804, + 0.012605960676272007, + 0.020910899279891242, + -0.03133534222539245, + 0.07765497927881118, + 0.002774123520475092 + ], + "11": [ + -0.23382872191020918, + -0.07868389546719848, + 0.005836266046029812, + -0.04401923559906355, + -0.08945417964085227, + 0.07278673444613164, + -1.2345209696385488, + -1.4020742160068793, + 0.5270645601651651, + 0.0036364677220707613, + 0.016239315538136664, + 0.5063895301634869, + 0.01137954639685464, + 0.1623105652474296, + -0.04715100881725161, + 0.005563556127206866, + 0.4442461428934573, + 0.1623803965923353, + 0.18428897378097156, + 0.02848508536008336 + ] + }, + "10": { + "0": [ + -0.01250766249447616, + 0.047268805772708936, + -0.005088008819595863, + -0.012344973964802447, + -0.015796338424422615, + -0.04659671574974606, + 0.00010884459423950673, + 0.19124402971119503, + -0.33270668620308674, + 0.3273259538000065, + 0.0006158078736545312, + 1.0371811385390912, + -0.008678009880180632, + -0.013655760653608283, + -0.0002379958234768884, + 0.0011722376179560193, + 0.005179492835026633, + 0.004162681363402172, + -0.032662862764620396, + -0.32375567525443644 + ], + "1": [ + -0.06255300528220267, + 0.03219988068859921, + 0.009150978907058583, + -0.40845677686985127, + -0.012678689336361758, + -0.003747118572940528, + -0.029541539231414837, + -0.05011170053990456, + -0.0011252132439596044, + -0.017993555021216415, + -0.019232772073635388, + 0.6670303031474882, + 0.029268236608206752, + -0.05255166683624879, + 0.009679888609834847, + 0.012975755919156288, + -0.058354478302946496, + -0.08682820749218124, + -0.09846279036115237, + -0.0821881495156527 + ], + "2": [ + -0.04157493792541632, + -0.025710503364795974, + 0.020277314795856295, + -0.14535856157929902, + 0.01651176210382144, + 0.06338640925705746, + 0.41312663465362726, + 0.41130026146531506, + -0.27258116112368, + -0.03732575731086161, + 0.010179716416857694, + -0.07966091487342954, + 0.04815417245480696, + -0.012077724139422718, + 0.0281591276755881, + 0.005517996821204036, + 0.2874424039629514, + -0.021918559313232517, + 0.4298422202000327, + -0.3476784796618142 + ], + "3": [ + -0.05151952137318784, + 0.009177520977929877, + -0.017173432967685847, + -0.07047395118873853, + -0.012909672548733108, + -0.1869160621755203, + -0.14557824935182437, + 0.5810769433707228, + -0.22730006418197646, + -0.08746782272871723, + -0.0026634823725651953, + 0.5113461514151675, + 0.004503197265485344, + -0.04398119538549107, + -0.0010549756385700667, + 0.0016863594214624037, + 0.09864996791975478, + -0.11381100423002323, + -0.07084051994144179, + -0.014684477184092128 + ], + "4": [ + 0.08565010232218814, + 0.024328520817399668, + 0.03375050527670422, + -0.017120178223488023, + -0.032444193339857444, + -0.0691833250073697, + 0.5197301466066129, + 0.1000729154926022, + 0.21216559660561296, + -0.032733930213459196, + 0.008413234945428082, + -0.11059908957183028, + 0.028676380551550255, + 0.17331203083460234, + 0.012241707200944744, + 0.0013395101064189632, + 0.057640333751646636, + 0.006907949422628102, + -0.3094168489074875, + 0.592524780202926 + ], + "5": [ + 0.2783201825852674, + 0.009773930866879754, + -0.00699285492345725, + -0.013111862529126094, + -0.011082616148174148, + -0.050269066732607184, + -0.012053841090267425, + -0.3984337437591234, + -0.6079191689811195, + -0.0023245171228129303, + -0.02888792940007986, + 0.024753796408688845, + 0.002481977011785308, + -0.09856593929390325, + 0.07754395694126313, + 0.08482477678369181, + -0.15823980738551627, + 0.09711072304589785, + 0.0465201009532401, + -0.257611352618503 + ], + "6": [ + -0.02035117413771974, + -0.0171434622338777, + -0.012977509533379025, + 0.07593120748929792, + 0.007351925128900685, + -0.12935472789101277, + 0.1989511729476276, + 0.5038199580615699, + -0.14189568007522885, + -0.04142941674342389, + -0.009025211557338006, + -0.5386271081992204, + -0.004535017483585156, + -0.044075968454627676, + -0.021886337170264496, + -0.002471509213233797, + -0.04232123298454774, + 0.006223722107301238, + 0.02582205886795706, + -0.37964069986564764 + ], + "7": [ + 0.11643725276949986, + -0.004787694062671025, + -0.00982872525561479, + 0.03413788064758121, + 0.0023776704365360665, + 0.1268825423800756, + 0.026962759614048065, + -0.5066155172392984, + -0.05521931577965735, + -0.08502126368969427, + -0.005748133729872208, + 0.007167886580124214, + 0.0050479593993541195, + -0.005193044884197665, + 0.0005655300366828985, + 0.001752869357232958, + 0.009521927350664762, + 0.006128790045227093, + -0.014730999405202505, + -0.0969630555421578 + ], + "8": [ + -0.05255618073284227, + 0.06330308631272048, + -0.021025572022290242, + -0.16388568213054402, + 0.218147620924962, + -0.18806185121829558, + -0.61055119239487, + 0.13967956446402366, + -0.21164492339645152, + -0.18066303444707238, + 0.01610570702997257, + 0.703677385240236, + 0.03460003235301119, + 0.02435473137627652, + 0.035727673219841424, + 0.04200468248557241, + 0.43901219459155594, + 0.002670635934953803, + 0.6398610468914324, + -1.5128891188660352 + ], + "9": [ + 0.0025434886336088047, + 0.03616526386899597, + -0.00932146789535999, + -0.003563973908461372, + 0.015471297183229779, + 0.003052848093631775, + 0.5115556384905227, + 0.10031021968781312, + 0.05153965880484535, + 0.0020194123322878534, + 0.011894125105420445, + 0.226565138291341, + 0.031222634403897182, + 0.0722975708777999, + -0.0394451401467816, + 0.03890398927994917, + 0.21280418519398248, + -0.0232440258403055, + -0.09736949115045328, + 0.009976343259752697 + ], + "10": [ + -0.008088391806941478, + 0.013438373277622899, + -0.01331918911075674, + 0.004171269014711094, + -0.002562040820464909, + 0.09624142455480705, + -0.3064254416545293, + 0.21341171428821448, + -0.19929553332144145, + 0.23428424792938427, + 0.01977071359943823, + -0.19473138485167077, + 0.003612867563052614, + 0.010607443306584256, + -0.0009826026396180304, + -0.0011403128487861532, + -0.012713017709523164, + 0.030826052609359455, + 0.009181078913942903, + 0.011046904071544874 + ], + "11": [ + -0.02329789877421775, + 0.012203343625537965, + 0.01266459558777233, + 0.07168434157339391, + 0.004944705488619815, + 0.12234113953849932, + 0.07667683031118482, + 0.03655642187688117, + 0.02473372466070211, + -0.07222593153704886, + 0.013634489243267094, + -0.46928977372515196, + 0.03891485392734565, + 0.03440132583536743, + 0.027882625705232884, + 0.00987772311096387, + 0.03747780817366222, + -0.047431998788404865, + -0.3783912958989409, + -0.016297611050362083 + ] + }, + "11": { + "0": [ + 0.20226938268003677, + 0.1640580559927482, + 0.002219163251017887, + -0.06826904157061194, + 0.20119511788077796, + 0.06618194031989648, + 0.7989974665548693, + 1.247768610047783, + -1.1748938547961685, + -0.25511146369260235, + -0.0032258684275272985, + -0.2367291428422577, + -0.0012849004068703942, + 0.02598534746991459, + 0.0219870098675247, + 0.0077327776823634935, + 0.7070560053832511, + -0.21913006683186673, + 0.48709333830567664, + -1.166112010568918 + ], + "1": [ + 0.03229316345405377, + -0.022161747275893696, + 0.04124254915150489, + -0.0022527876555487624, + -0.02025077684245964, + 0.07370336568035186, + -0.09584743333249691, + 0.024245543261915015, + -0.2821140236377229, + -0.07247764298923204, + 0.008314431561563602, + 0.460307933112711, + 0.06695546652126166, + -0.003973328467021738, + 0.003786592419528973, + -0.007154806340517377, + -0.03107851288064193, + -0.007462615370218552, + 0.15301359113916604, + -0.164253782535133 + ], + "2": [ + 0.03712138401496393, + -0.0001680578324039625, + 0.04538585252769399, + -0.04785535835181941, + 0.013190181891324619, + -0.16914565092820633, + 0.06176233001334061, + 0.5011401570278466, + -0.11606469777911273, + 0.0075780402346666, + -0.010934528941978584, + -0.0819698797009207, + 0.001180530091503012, + 0.03454997674517759, + 0.02737601471256863, + -0.0002876554722076473, + 0.08120554807405979, + -0.024205884657924677, + 0.09181957065919367, + -0.01695103236328156 + ], + "3": [ + 0.013419432970644657, + 0.0005119901405795137, + -0.007039160574600841, + 0.024234686591309805, + -0.005530695331897039, + -0.07700770475879617, + -0.12735654792131823, + 0.027909288519683505, + -0.21087963861251627, + -0.3611554009019448, + -0.00014823205651323145, + -0.4869864096526372, + 0.06587421551023007, + 0.03500501730315541, + 0.017423799497702723, + 0.004875178291981629, + 0.054732411820536765, + 0.016791033847240285, + 0.0020153587859874666, + 0.08699838052013581 + ], + "4": [ + 0.038034787025557085, + 0.030374694462258506, + 0.0006861655578550407, + 0.019042019444925597, + 0.042535454511691856, + 0.07053302565945245, + 0.10414474047798546, + 0.14857557783083275, + 0.11689637713334373, + 0.030209188071864186, + -0.018162716856796154, + 0.10608862936580027, + 0.023975897933846076, + -0.0943900820763978, + 0.023093017748945562, + -0.010627955186455722, + 0.170486063950399, + -0.06489412469710013, + 0.35440896542167827, + -0.16367037064859777 + ], + "5": [ + -0.11820120621533703, + -0.032739229080965424, + 0.10861130795884413, + 0.04378908503320958, + 0.013297557871129679, + 0.47067945768778985, + -0.2719217052806056, + -3.512663516119942, + 0.4816366962785972, + -0.03864914933976413, + 0.009702859233236499, + 0.7903466108453454, + -0.038751934410674614, + 0.2330378891783702, + 0.13914219370957476, + 0.010305381997968533, + 0.01620905868287998, + 0.14480841219671148, + 0.06545305756582824, + 0.10957058641018445 + ], + "6": [ + -0.036551629500760165, + -0.017539766052616347, + 0.019716384976321423, + -0.0693357978430008, + -0.03584235177050982, + 0.1605706822529957, + 0.1830654439627232, + 0.2545232191317051, + -0.17816808216311922, + 0.024713488032531237, + -0.006439271764026365, + 0.32461961218546187, + -0.0042034508109851185, + -0.006706817467941284, + 0.025032242977275763, + 0.0033677305977420162, + -0.027761397012730612, + 0.003948337902492622, + -0.12572501847051223, + 0.03350534464371915 + ], + "7": [ + 0.0026202177515674786, + 0.005533401839942561, + -0.001918176518584541, + -0.0956183204910697, + -0.02075352949208953, + 0.05780020216969626, + 0.07861370591124169, + -0.03877319033458312, + -0.12012175546929005, + -0.03663164391241706, + -0.025199665452981437, + -0.6085376130677014, + -0.06329168660924936, + 0.03673105559654061, + -0.030078403936258446, + 0.005986226769028738, + -0.01944838102668456, + -0.07688720853913389, + 0.11684251122856747, + -0.12758634546639283 + ], + "8": [ + -0.05386547334141368, + -0.05571781558877605, + -0.012417982915393958, + -0.030554587531037205, + -0.0398298295321853, + -0.0007923422533090103, + -0.4908974926812707, + 0.10788080230576128, + 0.40900453082611155, + 0.25647108691537973, + -0.045289075275851465, + 0.10163353220923056, + -0.03660534249766132, + 0.04759880027794527, + 0.00028160519464029486, + -0.003260649601151424, + -0.051122792301984105, + -0.13901277995423517, + -0.2634719374988321, + 0.2866477622113622 + ], + "9": [ + -0.026960489755925934, + 0.04712185287739756, + -0.05186653888547212, + 0.041527057756403904, + 0.013277164758686082, + -0.19945953917091955, + -0.07294541434969198, + 0.03795709541934559, + -0.2740243694148456, + 0.07561640852175877, + 0.02268026010658673, + -0.0743949118747649, + 0.0134981365179401, + 0.049568262550277235, + -0.020195314213981342, + 0.00328658847610194, + -0.0193830344664349, + -0.009795914103462339, + -0.07736079836669536, + 0.16986912194303475 + ], + "10": [ + 0.04696719913004397, + -0.00010005303510561488, + 0.06341629084192374, + 0.020831986072059976, + 0.013360818138301651, + -0.08584970054143325, + 0.16022761538164926, + 0.45997077311235335, + -0.27481411535625827, + -0.0572834244210832, + 0.024956245428616245, + 0.7840677897065541, + 0.06501634243025914, + 0.11085917479602017, + 0.00205659938688703, + 0.05154054452667563, + 0.003936352319800955, + -0.1505156800040496, + 0.10816636970109202, + -0.06573593431535879 + ], + "11": [ + 0.04857851060717612, + 0.03561028218989451, + -0.01487253322639177, + -0.06455639375647118, + -0.0012930898033113177, + -0.06481476152987309, + 0.17554121252503832, + -0.23752413568452338, + -0.028161781158603884, + 0.12490799433602574, + 0.011475600230896738, + -0.26036590831183587, + 0.03990891754078377, + -0.01924607347144707, + -0.004533518883072424, + 0.027068546209579034, + 0.02579788846427654, + 0.02621378732934199, + 0.06602385635053057, + -0.04204357760316246 + ] + } + } +} \ No newline at end of file diff --git a/data/mi2b_depth/gpt2-large.json b/data/mi2b_depth/gpt2-large.json new file mode 100644 index 0000000000000000000000000000000000000000..7bbcf61f94d183388c26646fa5da675693a0e158 --- /dev/null +++ b/data/mi2b_depth/gpt2-large.json @@ -0,0 +1,1866 @@ +{ + "model": "gpt2-large", + "N": 36, + "L_crit": 33, + "N_sem": 3, + "n_heads": 20, + "n_pairs_signal": 9, + "R_mean_matrix": [ + [ + 0.0007416779143460503, + -0.0024345251971012896, + 0.00572834261352486, + -0.02542585768945263, + -0.006572931451709801, + 0.0009652734740854791, + 0.014907738095520345, + -0.005825448553651125, + -0.018488172217377283, + 0.003677585372727466, + -0.0044836922362377065, + -0.017735530984097244, + 0.00638277188048307, + -0.05666008801844902, + -0.08636581271690554, + -0.010609085078044953, + 0.0016683970157402713, + 0.0019525260619940449, + 0.009808292182804552, + -0.012447681724973712 + ], + [ + 0.0001603049891998131, + -0.0083370234950861, + -0.0027457046418088416, + -0.057148475378393765, + 0.006616223746633242, + 0.0020149991282826926, + -0.005701583698820654, + -0.004070481586513009, + 0.00456296644904526, + -0.013997754280840602, + -0.010140700918393765, + -0.020565739789262464, + -0.002177688216335423, + 0.0054820152415950875, + -0.01276565679863463, + 0.0023644401769287128, + 0.0007278429285695625, + 0.0048860274928291016, + 0.0074499385403891056, + -5.244875940076592e-05 + ], + [ + 0.004308841545018132, + -0.006072530720243086, + -0.03441345354384056, + -0.01675674604446275, + 0.023905745523025327, + -0.010390875516216877, + -0.0035222958221927283, + 0.002562822036187457, + -0.0064682328494794356, + 0.023256043949162888, + 0.013660829641554621, + 0.014872864864600675, + 0.010716414096893244, + -0.004096049943217137, + -0.020734522079388164, + 0.015313826218869135, + 0.001000154908414454, + -0.12797746059738357, + 0.0028223711165181295, + -0.01679210139011011 + ], + [ + -0.023877797464344427, + -0.002685264511696443, + -0.00394141132015061, + -0.008858655296103378, + -0.02332880973984457, + -0.018441798210897017, + -0.0047628197815180705, + -0.0020125524968069705, + 2.5863803546493816e-05, + 0.011915301260764503, + -0.01923852008661423, + -0.016489713585913123, + -0.0017535622690354345, + -0.11252165531456776, + 0.021022046514935184, + 0.011865228427006306, + -0.0016645211474998662, + -0.011049639781547476, + 0.0407458822889244, + 0.017172269779017795 + ], + [ + -0.015526249900643628, + 0.006205985950339845, + -0.0075624057468247664, + -0.04993569004842982, + -0.008338438044985883, + 0.031704730722405046, + 0.0340502664948661, + -0.004170278704467586, + 0.0015889583125791177, + -0.0047553043850361715, + 0.009631538196220164, + 0.016879782268269646, + 0.0003160508588478983, + 0.00373156433042054, + 0.004897794427530103, + 0.029496115148961513, + 0.0006910374291605079, + -0.1661604511284519, + -0.006079179571440956, + -0.0037259705080080145 + ], + [ + 0.0150069197158996, + -0.013714931907491974, + 0.030032956379413316, + -0.009632167446382251, + 0.007722365888107961, + 0.014524624867171832, + 0.026482404946205518, + 0.02797308336477718, + 0.02252254780730564, + -0.02068191872579826, + 0.010865141858054537, + 0.014660545779661563, + -0.017850895933870705, + -0.007863852846063328, + 0.009270342825271243, + -0.014109642708089731, + 0.01024930390081732, + -0.014896111294383035, + -0.030518326585441864, + 0.01749088767419829 + ], + [ + -0.051346129227083086, + 0.07923933405753308, + 0.00017188692815028382, + 0.013911167526141901, + -0.03495568982419039, + -0.00636462314912585, + 0.011726489027420154, + -0.004820099144668653, + -0.02062581951319918, + 0.0018531488008733492, + -0.02366375359144439, + -0.015293411789345259, + 0.13698115120069004, + 0.011302694516486815, + 0.007065237527548434, + -0.006667931820120298, + 0.029182917737123597, + 0.005775584691895019, + -0.02194200372741237, + -0.011921029800284363 + ], + [ + 0.00674694216275881, + 0.007741029677816733, + 0.006656656072449387, + 0.004658604292103297, + -0.008805724091183813, + -0.06505190846585489, + -0.007740659783468452, + -0.002160083528691355, + -0.007589816956562355, + -0.0010413422974027073, + 0.013377185426541729, + 0.0018376238015533361, + 0.008500358275961111, + 0.09154138524216585, + 0.000884388450414089, + 0.014929485954120993, + -0.019956705359190155, + 0.0060664136085450794, + 0.005293035407553851, + -0.03164238198386851 + ], + [ + -0.008954092883529394, + 0.01078529639307287, + -0.024116539229663003, + -0.0007738623298291759, + 0.026377814203506652, + 0.010794850857339479, + 0.006200530373447508, + 0.0008636327172242967, + 0.040887215499667055, + -0.00010590631329054623, + 0.030627445351202615, + -0.009614038991575978, + 0.01591590919333859, + 0.043734513526017385, + 0.008542556309406146, + -0.008737375543360258, + -0.003860834772656911, + -0.0017458241524243355, + 0.018262717449975773, + -0.01965595392017487 + ], + [ + -0.005191813642093708, + 0.002391268028641546, + 0.00896869494350682, + -0.012396231667423043, + -0.030831616842360656, + -0.004748458679399921, + 0.004829955703301965, + -0.0002814140706357918, + 0.005504295158013265, + 0.0013486574435772208, + -0.022222890909315843, + -0.025695440102721165, + 0.026324775822957572, + -0.020154532635648814, + -0.007187203172278204, + 0.045099762320618283, + -0.008333181454781496, + 0.0192690735708806, + -0.0023852993834283386, + -0.0009235798032036785 + ], + [ + 0.00965900687065741, + -0.00017859929905176633, + 1.645256297751276e-05, + 0.05724407200569553, + -0.0002543086688476948, + -0.021673500551492305, + -0.007928301630775939, + -0.027773887534120707, + -0.005547864474626577, + 0.0009001180813858931, + 0.022161926664070804, + 0.0032092593226517078, + -0.03767395797844804, + -0.0030333177872682465, + 0.043918979201380305, + 0.018135401717892963, + -0.0024422803841826464, + 0.038510367693229706, + -0.028674287160596876, + -0.11870330118505386 + ], + [ + -0.055106988556313304, + 0.0015787173466733336, + -0.026677032218835044, + 0.0010708578391049447, + -0.007227151688499451, + 0.015958114503835427, + -0.024558619950138656, + 0.010042352163738938, + -0.00017402047645273042, + -0.000359914925890041, + 0.0026203688860061015, + 0.007762083955290984, + -0.0026352685218601086, + -0.025059504680825722, + -0.0005371476285010346, + -0.004468582221044813, + -0.0024817399685882497, + -0.003640731143676747, + -0.005016351475418439, + 0.033632651096011934 + ], + [ + 0.02006612983384283, + 0.022629920018842904, + -0.0009486377008795674, + -0.01882567133063986, + 0.011698453412585332, + 0.0025416077239399713, + -0.06576265851144135, + -0.022429197763047824, + 0.0036344970065091686, + -0.0004842402882827486, + -0.005384374926714448, + -0.0033592492642385403, + -0.03095665259209039, + 0.10041905713738701, + 0.10895724721023614, + -0.0161851553021354, + -0.006699442364365345, + 1.7372768958077256e-05, + 5.381858027707096e-05, + -0.0076494039968532 + ], + [ + -0.09523420846330125, + 0.0019943238226171217, + 0.014936843497412953, + 0.02031605249210021, + 0.02269058113708039, + -0.005693560619452895, + 0.012529909176517954, + 0.0004815850432337129, + 0.00013488782573805601, + 0.01641082323436807, + 0.010510845376180876, + -0.0029987358752987512, + -0.014093684636878786, + 0.006171902269223676, + -0.002841565760206218, + 0.07280113988176408, + 0.00828182151698119, + -0.010948894467866973, + -0.0089869244919433, + -0.0014085985775011928 + ], + [ + -0.02154220772490503, + 0.03659703782010214, + -0.016680195712594985, + -0.03857730671959106, + -0.0015902756858942617, + 0.004181589777216168, + -0.02721600498007854, + 0.04948693632875997, + 0.004631399098086023, + 0.003452637321262454, + 0.0032595597590014246, + -0.00010523916191276525, + -0.003140348499700924, + 0.006817927064421661, + -0.00587564301139575, + 0.015677657594006654, + -0.0018756569127196428, + 0.01437237290672357, + 0.006530774769171198, + 0.02884989916568413 + ], + [ + 0.011221908609700196, + -0.012363775941901098, + 0.0014027905157225026, + 0.0005199788004135217, + -0.037543957044218396, + 0.0012091007471348686, + -0.009854707535611443, + -0.02852193866165624, + -0.009984337133886391, + 0.005432210284418746, + -0.007597999417878852, + 0.05333245247130691, + 0.051846403324213725, + 0.03676584151387159, + -0.0043409321861559574, + 0.0002812050523614787, + -0.016025437233482705, + -0.006895506997197003, + 0.022208256123245687, + -0.031082010902406174 + ], + [ + -0.03822352611555853, + -0.02882258984896135, + -0.01868034545905224, + -0.0031216472583112066, + 0.012260791206821067, + 0.007268416097369186, + 0.012260092216244831, + 0.02442595841586037, + 0.0008527148712198564, + -0.026843854293622994, + -0.049699302022289046, + -0.013506463563240718, + -0.006748396503212761, + -0.0019477877418103877, + -0.016364551327912605, + -0.06597988011055436, + -0.030397150302079845, + -0.004032249190012251, + -0.013297021085350207, + 0.003164975751439464 + ], + [ + -0.008156373973767601, + -0.008228964366076128, + 0.027793920585129694, + -0.024632279471557706, + -0.04322377640691773, + 0.03356688601313608, + 0.027503048096218837, + -0.0049179488722540885, + -0.002947505436550811, + 0.015366057233975165, + 0.008007563700779133, + 0.08108031504115559, + 0.0288820357317065, + -0.004082867981730795, + -0.029070614845599894, + 0.021327162590875352, + -0.01466953825744171, + -0.021620237669663332, + -0.0020978754899474675, + -0.0003066856561057734 + ], + [ + 0.004219044517331905, + 0.0013821663427284257, + 0.00017595505510895952, + 0.00519611971987838, + -0.02589983256825769, + 0.004623642103451903, + -0.029252092991576384, + 0.02852031078890587, + 0.042658492151948266, + 0.0021677544867069708, + 0.009270521275141044, + 0.001377625473164957, + -0.001528685466647208, + 0.011156149954937617, + -0.00922221417340671, + 0.006016782868365515, + -0.011400023282046352, + 0.03311556560395442, + -0.08214506502488715, + 0.01924326533873281 + ], + [ + 0.0003258313406875405, + -0.011935299374063795, + -0.01269204196189125, + 0.12646392646285057, + -0.0027103635203999804, + 0.00013980544890245946, + -0.02281778306005, + 0.013161591164425075, + 0.00949845763492089, + -0.028472163355863032, + -0.014774241570226807, + 0.002180502550231458, + 0.008051334477220954, + -0.06863061086553576, + 0.061645770099245545, + -0.014776875637584976, + 0.006847486876011203, + -0.00690127203702081, + -0.001142902044897442, + -0.06496626444388715 + ], + [ + -0.11390571590788784, + 0.02015633490301224, + -0.00416299393784836, + 0.003244817669163446, + -0.027378320177853497, + 0.008269544300468525, + 0.003063787169846127, + -0.09378750731134505, + 0.0209807652639036, + 0.01130665842325247, + 0.05829035084574953, + -0.006958309856889389, + 0.000803602363585064, + -0.00432589993611309, + 0.029727440014830464, + 0.0003623514295813302, + 0.009987655282125455, + 0.0012541225849077113, + -0.01603133471996518, + 0.022808408072068798 + ], + [ + 0.03313016777862248, + -0.0012987182778222651, + 0.00430287902795317, + 0.00342636225023412, + 0.017297236817815895, + -0.01873052345698283, + -0.0036737995360301137, + -0.004724434489599887, + 0.013576304313864153, + 0.0032191985014994482, + -2.6994408653779035e-05, + -0.02454287405887143, + 0.030562864606032674, + -0.004229246117975251, + -0.0007342938338729212, + 0.004386594052135186, + -0.011437243413873062, + -0.009052305236256953, + -0.011058861099482474, + 0.007886721942471088 + ], + [ + -0.003902890686369654, + -0.012302295311217017, + 0.24964245586518252, + -0.011241033410484994, + -0.04168578746559944, + -0.010051371604635116, + -0.018861116167645225, + -0.0044737989001879365, + -0.006807800641358651, + -0.008479666128685365, + -0.008206531677135021, + 0.03762290382788439, + 0.024528784889528095, + -0.03766365803480778, + -0.0037544474949862366, + 0.012560194383849276, + -0.031013522083832092, + 0.010194119743130054, + 0.0008257281487037992, + 0.007255667009726534 + ], + [ + 0.046698124146367204, + 0.002848318605949574, + 0.10691937901687815, + -0.023823924535009373, + -0.003150488544639758, + 0.001319208147842292, + 0.015672247918876774, + -0.023080462420765533, + -0.010107916438924601, + -0.011974983091593784, + 0.029309628170211464, + -0.016991777812522046, + -0.004844922012130719, + 0.010125237104633871, + -0.0006617571673224413, + -0.02423516527453047, + -0.013964349950406511, + -0.002420840662100865, + 0.0002485366533294013, + -0.023084525169239933 + ], + [ + 0.0008852543161235271, + 0.021516871819612062, + -0.009138629668545898, + -0.008303075932337829, + 0.012235145476287916, + 0.050050267191613346, + -0.008732376604644513, + -0.012716547307187239, + 0.002576200231005693, + -0.004307708316967207, + -0.019522285183458304, + 0.010061023331861884, + 0.007049417552930979, + 0.0027477871841884082, + 0.044039574504288095, + 0.0024504735992134747, + 0.005685471276692106, + 0.00031402951255964307, + 0.0006793394870351967, + 0.0002181863167802908 + ], + [ + -0.024193195044325506, + -0.010671325690029779, + -0.0013381953604193445, + 0.005897602527022623, + 0.06020661452424007, + 0.014527303134404417, + 0.02065137595607349, + -0.012663709878624925, + -0.046780057023669684, + 0.006424133029543689, + 0.025522093711375107, + -0.0005934135320710485, + 0.0021422948848589104, + 0.0047871872473510034, + -0.004617123420132598, + 0.006315300971410217, + -0.0012378320005630192, + -0.01473426702251353, + 0.0033234840091290506, + 0.012533357341527001 + ], + [ + -0.009253817072406556, + -0.0012105988306141515, + -0.006269260201747059, + -0.006841658722166846, + -0.003959710975619098, + -0.0023335817948334442, + 0.020165338455254575, + 0.005679334952391001, + -0.026168654641640976, + -0.006325568107612662, + 0.08242937420179375, + 0.0502946123505352, + 0.04584784048933921, + 0.028219194483755896, + -0.0040434391297670255, + -0.0038467902591241537, + -0.003447735180757004, + 0.004307525944967943, + -0.015796151970064336, + 0.0045441022722234456 + ], + [ + -0.004928663384639123, + -0.0009260684464156908, + 0.004896857481571466, + 0.04508823875232697, + -0.004429143238242332, + 0.10554927295816059, + 0.007325029915908701, + 0.004798999261253804, + -0.0005989735633548239, + 0.0041544229434214396, + 0.05772803954347377, + 0.00868832465994504, + 0.0036004131335843647, + -0.002122681282522863, + -0.015307353338176356, + 0.008540857005266175, + 0.015609116561321888, + 0.012366264862508816, + 0.000642768081016579, + 0.039207340921588923 + ], + [ + 0.0020271732956726467, + -0.0030698900870179315, + -0.0035581365842041073, + -0.00893913089593138, + 0.01890297642156609, + 0.0020060768472459833, + -0.003025066195656319, + 0.04344987167310966, + 0.009114221143592881, + -0.003929284876915595, + 0.0005746845818707819, + -0.0020465353573950037, + 0.013405278000484014, + 0.0007283502257467175, + -0.02313991882347066, + -0.00785336096465147, + 0.013174167174064302, + 0.02833920913970194, + 0.009347077739723701, + -0.016827772775230662 + ], + [ + 0.0006583682555183891, + -0.010659840809871323, + -0.00760570579277298, + -0.007166254349327647, + 0.05788074313123995, + -0.0028388159466482313, + 0.03323945339814239, + 0.00023790630596653617, + -0.0016789330217736366, + 0.005566385540787174, + -0.011593325397984454, + 0.00012693501697462784, + -0.0052495853424805354, + -0.0001214190770435006, + -0.008300949235705405, + 0.005781335447203732, + 0.012210697800788976, + -0.0009492296092764867, + -0.01274835966462409, + -0.0034564456110690368 + ], + [ + 0.008689560986910459, + 0.0006236555372798104, + 0.0024674913206588138, + 0.014136069762919463, + -0.00018910199534941138, + 0.015728739274268797, + 0.004305280190829549, + -0.0016840241859316695, + 0.002315347800672893, + -0.011318182190661425, + -0.0032914744612618733, + -0.004697394549137992, + -0.008761179987624811, + -0.0017935324358357822, + 0.004622968465028163, + -0.007683186520120099, + 0.0068293298671813695, + 0.003787489772402743, + 0.0038747058788893623, + 0.05614767511181293 + ], + [ + 0.0005978890060906091, + 0.006387300568594515, + 0.08264338381290322, + 0.019959766600075337, + -0.01795513555517919, + -0.0048787135277967575, + -0.006921832985989966, + 0.00966178953341835, + -0.00626567113164959, + -0.0225686455915563, + 0.013722326384865667, + -0.0021761941825296236, + -0.000758208634952818, + 0.013417226259867075, + 0.015962416131708568, + -0.0011242556437812438, + 0.009620825318704299, + 0.01744185044761553, + 0.04204499934558952, + -0.011898693767065554 + ], + [ + -0.004955536806871526, + 0.009420486673214226, + 0.015966848314120256, + -0.003810604520496753, + 0.03701845705858334, + 0.07447903043432628, + -0.010096120546969421, + -0.004524966871906906, + -0.026979142916085073, + -0.0008730748147393633, + 0.013742700890831544, + 0.011823511560561363, + 0.012785011141264865, + -0.008336916659661332, + -0.004708330328498493, + -0.0008219581106703905, + 0.02000343511115649, + 0.0003760428523318079, + 0.02900020357010437, + -0.008145703878699588 + ], + [ + -0.0027014760170596345, + -0.003883317483111803, + 0.005563778721309287, + -0.028049418857555235, + 0.020761589081275345, + -0.001783668761694085, + 0.005192200683497365, + -0.01593003836363109, + 0.004869322332706283, + -0.010395914855515968, + -0.00034846510463167583, + 0.019189931782176112, + 0.023452344471991432, + -0.0008954856803347184, + -0.015065360978749316, + 0.005941110305233834, + 0.006748258156652932, + -0.0008439442524162444, + 0.0019679637601425558, + -0.0012240415701286703 + ], + [ + -0.010342740014519132, + 0.0007578434589704537, + -0.0009429298253037832, + 0.018682523725480252, + 0.030279022120833388, + -0.0180565433868319, + -0.008948553628467307, + -0.0013383974427371697, + -0.008576944566073576, + 0.008941695926131216, + 0.0046089204994549104, + 0.005085829735551713, + -0.002305296984441929, + -0.0012001754896556584, + 0.005629833115608552, + 0.013762321845950033, + 0.006165380845668918, + 0.0050475607221486885, + -0.0008974492602818036, + -0.010240829932160751 + ], + [ + 0.023320029631584235, + -0.019556258255729705, + -0.016981603889820618, + -0.007784225312570819, + -0.002006816598813663, + -0.006883139481267061, + 0.002626040762994337, + 0.00470262336803242, + 0.0029746774170483687, + 0.0008230929972932282, + 0.007540530247723191, + -0.006089729354042471, + -0.003479402083298529, + -0.011484419440456261, + -0.0027402455696341954, + 0.023625091909264272, + -0.01117509118279306, + 0.045141556754060286, + 0.007946822058321153, + 0.009741510602046071 + ] + ], + "head_profiles": { + "0": { + "top_head": 6, + "top_R": 0.014907738095520345, + "per_head": [ + 0.0007416779143460503, + -0.0024345251971012896, + 0.00572834261352486, + -0.02542585768945263, + -0.006572931451709801, + 0.0009652734740854791, + 0.014907738095520345, + -0.005825448553651125, + -0.018488172217377283, + 0.003677585372727466, + -0.0044836922362377065, + -0.017735530984097244, + 0.00638277188048307, + -0.05666008801844902, + -0.08636581271690554, + -0.010609085078044953, + 0.0016683970157402713, + 0.0019525260619940449, + 0.009808292182804552, + -0.012447681724973712 + ] + }, + "1": { + "top_head": 18, + "top_R": 0.0074499385403891056, + "per_head": [ + 0.0001603049891998131, + -0.0083370234950861, + -0.0027457046418088416, + -0.057148475378393765, + 0.006616223746633242, + 0.0020149991282826926, + -0.005701583698820654, + -0.004070481586513009, + 0.00456296644904526, + -0.013997754280840602, + -0.010140700918393765, + -0.020565739789262464, + -0.002177688216335423, + 0.0054820152415950875, + -0.01276565679863463, + 0.0023644401769287128, + 0.0007278429285695625, + 0.0048860274928291016, + 0.0074499385403891056, + -5.244875940076592e-05 + ] + }, + "2": { + "top_head": 4, + "top_R": 0.023905745523025327, + "per_head": [ + 0.004308841545018132, + -0.006072530720243086, + -0.03441345354384056, + -0.01675674604446275, + 0.023905745523025327, + -0.010390875516216877, + -0.0035222958221927283, + 0.002562822036187457, + -0.0064682328494794356, + 0.023256043949162888, + 0.013660829641554621, + 0.014872864864600675, + 0.010716414096893244, + -0.004096049943217137, + -0.020734522079388164, + 0.015313826218869135, + 0.001000154908414454, + -0.12797746059738357, + 0.0028223711165181295, + -0.01679210139011011 + ] + }, + "3": { + "top_head": 18, + "top_R": 0.0407458822889244, + "per_head": [ + -0.023877797464344427, + -0.002685264511696443, + -0.00394141132015061, + -0.008858655296103378, + -0.02332880973984457, + -0.018441798210897017, + -0.0047628197815180705, + -0.0020125524968069705, + 2.5863803546493816e-05, + 0.011915301260764503, + -0.01923852008661423, + -0.016489713585913123, + -0.0017535622690354345, + -0.11252165531456776, + 0.021022046514935184, + 0.011865228427006306, + -0.0016645211474998662, + -0.011049639781547476, + 0.0407458822889244, + 0.017172269779017795 + ] + }, + "4": { + "top_head": 6, + "top_R": 0.0340502664948661, + "per_head": [ + -0.015526249900643628, + 0.006205985950339845, + -0.0075624057468247664, + -0.04993569004842982, + -0.008338438044985883, + 0.031704730722405046, + 0.0340502664948661, + -0.004170278704467586, + 0.0015889583125791177, + -0.0047553043850361715, + 0.009631538196220164, + 0.016879782268269646, + 0.0003160508588478983, + 0.00373156433042054, + 0.004897794427530103, + 0.029496115148961513, + 0.0006910374291605079, + -0.1661604511284519, + -0.006079179571440956, + -0.0037259705080080145 + ] + }, + "5": { + "top_head": 2, + "top_R": 0.030032956379413316, + "per_head": [ + 0.0150069197158996, + -0.013714931907491974, + 0.030032956379413316, + -0.009632167446382251, + 0.007722365888107961, + 0.014524624867171832, + 0.026482404946205518, + 0.02797308336477718, + 0.02252254780730564, + -0.02068191872579826, + 0.010865141858054537, + 0.014660545779661563, + -0.017850895933870705, + -0.007863852846063328, + 0.009270342825271243, + -0.014109642708089731, + 0.01024930390081732, + -0.014896111294383035, + -0.030518326585441864, + 0.01749088767419829 + ] + }, + "6": { + "top_head": 12, + "top_R": 0.13698115120069004, + "per_head": [ + -0.051346129227083086, + 0.07923933405753308, + 0.00017188692815028382, + 0.013911167526141901, + -0.03495568982419039, + -0.00636462314912585, + 0.011726489027420154, + -0.004820099144668653, + -0.02062581951319918, + 0.0018531488008733492, + -0.02366375359144439, + -0.015293411789345259, + 0.13698115120069004, + 0.011302694516486815, + 0.007065237527548434, + -0.006667931820120298, + 0.029182917737123597, + 0.005775584691895019, + -0.02194200372741237, + -0.011921029800284363 + ] + }, + "7": { + "top_head": 13, + "top_R": 0.09154138524216585, + "per_head": [ + 0.00674694216275881, + 0.007741029677816733, + 0.006656656072449387, + 0.004658604292103297, + -0.008805724091183813, + -0.06505190846585489, + -0.007740659783468452, + -0.002160083528691355, + -0.007589816956562355, + -0.0010413422974027073, + 0.013377185426541729, + 0.0018376238015533361, + 0.008500358275961111, + 0.09154138524216585, + 0.000884388450414089, + 0.014929485954120993, + -0.019956705359190155, + 0.0060664136085450794, + 0.005293035407553851, + -0.03164238198386851 + ] + }, + "8": { + "top_head": 13, + "top_R": 0.043734513526017385, + "per_head": [ + -0.008954092883529394, + 0.01078529639307287, + -0.024116539229663003, + -0.0007738623298291759, + 0.026377814203506652, + 0.010794850857339479, + 0.006200530373447508, + 0.0008636327172242967, + 0.040887215499667055, + -0.00010590631329054623, + 0.030627445351202615, + -0.009614038991575978, + 0.01591590919333859, + 0.043734513526017385, + 0.008542556309406146, + -0.008737375543360258, + -0.003860834772656911, + -0.0017458241524243355, + 0.018262717449975773, + -0.01965595392017487 + ] + }, + "9": { + "top_head": 15, + "top_R": 0.045099762320618283, + "per_head": [ + -0.005191813642093708, + 0.002391268028641546, + 0.00896869494350682, + -0.012396231667423043, + -0.030831616842360656, + -0.004748458679399921, + 0.004829955703301965, + -0.0002814140706357918, + 0.005504295158013265, + 0.0013486574435772208, + -0.022222890909315843, + -0.025695440102721165, + 0.026324775822957572, + -0.020154532635648814, + -0.007187203172278204, + 0.045099762320618283, + -0.008333181454781496, + 0.0192690735708806, + -0.0023852993834283386, + -0.0009235798032036785 + ] + }, + "10": { + "top_head": 3, + "top_R": 0.05724407200569553, + "per_head": [ + 0.00965900687065741, + -0.00017859929905176633, + 1.645256297751276e-05, + 0.05724407200569553, + -0.0002543086688476948, + -0.021673500551492305, + -0.007928301630775939, + -0.027773887534120707, + -0.005547864474626577, + 0.0009001180813858931, + 0.022161926664070804, + 0.0032092593226517078, + -0.03767395797844804, + -0.0030333177872682465, + 0.043918979201380305, + 0.018135401717892963, + -0.0024422803841826464, + 0.038510367693229706, + -0.028674287160596876, + -0.11870330118505386 + ] + }, + "11": { + "top_head": 19, + "top_R": 0.033632651096011934, + "per_head": [ + -0.055106988556313304, + 0.0015787173466733336, + -0.026677032218835044, + 0.0010708578391049447, + -0.007227151688499451, + 0.015958114503835427, + -0.024558619950138656, + 0.010042352163738938, + -0.00017402047645273042, + -0.000359914925890041, + 0.0026203688860061015, + 0.007762083955290984, + -0.0026352685218601086, + -0.025059504680825722, + -0.0005371476285010346, + -0.004468582221044813, + -0.0024817399685882497, + -0.003640731143676747, + -0.005016351475418439, + 0.033632651096011934 + ] + }, + "12": { + "top_head": 14, + "top_R": 0.10895724721023614, + "per_head": [ + 0.02006612983384283, + 0.022629920018842904, + -0.0009486377008795674, + -0.01882567133063986, + 0.011698453412585332, + 0.0025416077239399713, + -0.06576265851144135, + -0.022429197763047824, + 0.0036344970065091686, + -0.0004842402882827486, + -0.005384374926714448, + -0.0033592492642385403, + -0.03095665259209039, + 0.10041905713738701, + 0.10895724721023614, + -0.0161851553021354, + -0.006699442364365345, + 1.7372768958077256e-05, + 5.381858027707096e-05, + -0.0076494039968532 + ] + }, + "13": { + "top_head": 15, + "top_R": 0.07280113988176408, + "per_head": [ + -0.09523420846330125, + 0.0019943238226171217, + 0.014936843497412953, + 0.02031605249210021, + 0.02269058113708039, + -0.005693560619452895, + 0.012529909176517954, + 0.0004815850432337129, + 0.00013488782573805601, + 0.01641082323436807, + 0.010510845376180876, + -0.0029987358752987512, + -0.014093684636878786, + 0.006171902269223676, + -0.002841565760206218, + 0.07280113988176408, + 0.00828182151698119, + -0.010948894467866973, + -0.0089869244919433, + -0.0014085985775011928 + ] + }, + "14": { + "top_head": 7, + "top_R": 0.04948693632875997, + "per_head": [ + -0.02154220772490503, + 0.03659703782010214, + -0.016680195712594985, + -0.03857730671959106, + -0.0015902756858942617, + 0.004181589777216168, + -0.02721600498007854, + 0.04948693632875997, + 0.004631399098086023, + 0.003452637321262454, + 0.0032595597590014246, + -0.00010523916191276525, + -0.003140348499700924, + 0.006817927064421661, + -0.00587564301139575, + 0.015677657594006654, + -0.0018756569127196428, + 0.01437237290672357, + 0.006530774769171198, + 0.02884989916568413 + ] + }, + "15": { + "top_head": 11, + "top_R": 0.05333245247130691, + "per_head": [ + 0.011221908609700196, + -0.012363775941901098, + 0.0014027905157225026, + 0.0005199788004135217, + -0.037543957044218396, + 0.0012091007471348686, + -0.009854707535611443, + -0.02852193866165624, + -0.009984337133886391, + 0.005432210284418746, + -0.007597999417878852, + 0.05333245247130691, + 0.051846403324213725, + 0.03676584151387159, + -0.0043409321861559574, + 0.0002812050523614787, + -0.016025437233482705, + -0.006895506997197003, + 0.022208256123245687, + -0.031082010902406174 + ] + }, + "16": { + "top_head": 7, + "top_R": 0.02442595841586037, + "per_head": [ + -0.03822352611555853, + -0.02882258984896135, + -0.01868034545905224, + -0.0031216472583112066, + 0.012260791206821067, + 0.007268416097369186, + 0.012260092216244831, + 0.02442595841586037, + 0.0008527148712198564, + -0.026843854293622994, + -0.049699302022289046, + -0.013506463563240718, + -0.006748396503212761, + -0.0019477877418103877, + -0.016364551327912605, + -0.06597988011055436, + -0.030397150302079845, + -0.004032249190012251, + -0.013297021085350207, + 0.003164975751439464 + ] + }, + "17": { + "top_head": 11, + "top_R": 0.08108031504115559, + "per_head": [ + -0.008156373973767601, + -0.008228964366076128, + 0.027793920585129694, + -0.024632279471557706, + -0.04322377640691773, + 0.03356688601313608, + 0.027503048096218837, + -0.0049179488722540885, + -0.002947505436550811, + 0.015366057233975165, + 0.008007563700779133, + 0.08108031504115559, + 0.0288820357317065, + -0.004082867981730795, + -0.029070614845599894, + 0.021327162590875352, + -0.01466953825744171, + -0.021620237669663332, + -0.0020978754899474675, + -0.0003066856561057734 + ] + }, + "18": { + "top_head": 8, + "top_R": 0.042658492151948266, + "per_head": [ + 0.004219044517331905, + 0.0013821663427284257, + 0.00017595505510895952, + 0.00519611971987838, + -0.02589983256825769, + 0.004623642103451903, + -0.029252092991576384, + 0.02852031078890587, + 0.042658492151948266, + 0.0021677544867069708, + 0.009270521275141044, + 0.001377625473164957, + -0.001528685466647208, + 0.011156149954937617, + -0.00922221417340671, + 0.006016782868365515, + -0.011400023282046352, + 0.03311556560395442, + -0.08214506502488715, + 0.01924326533873281 + ] + }, + "19": { + "top_head": 3, + "top_R": 0.12646392646285057, + "per_head": [ + 0.0003258313406875405, + -0.011935299374063795, + -0.01269204196189125, + 0.12646392646285057, + -0.0027103635203999804, + 0.00013980544890245946, + -0.02281778306005, + 0.013161591164425075, + 0.00949845763492089, + -0.028472163355863032, + -0.014774241570226807, + 0.002180502550231458, + 0.008051334477220954, + -0.06863061086553576, + 0.061645770099245545, + -0.014776875637584976, + 0.006847486876011203, + -0.00690127203702081, + -0.001142902044897442, + -0.06496626444388715 + ] + }, + "20": { + "top_head": 10, + "top_R": 0.05829035084574953, + "per_head": [ + -0.11390571590788784, + 0.02015633490301224, + -0.00416299393784836, + 0.003244817669163446, + -0.027378320177853497, + 0.008269544300468525, + 0.003063787169846127, + -0.09378750731134505, + 0.0209807652639036, + 0.01130665842325247, + 0.05829035084574953, + -0.006958309856889389, + 0.000803602363585064, + -0.00432589993611309, + 0.029727440014830464, + 0.0003623514295813302, + 0.009987655282125455, + 0.0012541225849077113, + -0.01603133471996518, + 0.022808408072068798 + ] + }, + "21": { + "top_head": 0, + "top_R": 0.03313016777862248, + "per_head": [ + 0.03313016777862248, + -0.0012987182778222651, + 0.00430287902795317, + 0.00342636225023412, + 0.017297236817815895, + -0.01873052345698283, + -0.0036737995360301137, + -0.004724434489599887, + 0.013576304313864153, + 0.0032191985014994482, + -2.6994408653779035e-05, + -0.02454287405887143, + 0.030562864606032674, + -0.004229246117975251, + -0.0007342938338729212, + 0.004386594052135186, + -0.011437243413873062, + -0.009052305236256953, + -0.011058861099482474, + 0.007886721942471088 + ] + }, + "22": { + "top_head": 2, + "top_R": 0.24964245586518252, + "per_head": [ + -0.003902890686369654, + -0.012302295311217017, + 0.24964245586518252, + -0.011241033410484994, + -0.04168578746559944, + -0.010051371604635116, + -0.018861116167645225, + -0.0044737989001879365, + -0.006807800641358651, + -0.008479666128685365, + -0.008206531677135021, + 0.03762290382788439, + 0.024528784889528095, + -0.03766365803480778, + -0.0037544474949862366, + 0.012560194383849276, + -0.031013522083832092, + 0.010194119743130054, + 0.0008257281487037992, + 0.007255667009726534 + ] + }, + "23": { + "top_head": 2, + "top_R": 0.10691937901687815, + "per_head": [ + 0.046698124146367204, + 0.002848318605949574, + 0.10691937901687815, + -0.023823924535009373, + -0.003150488544639758, + 0.001319208147842292, + 0.015672247918876774, + -0.023080462420765533, + -0.010107916438924601, + -0.011974983091593784, + 0.029309628170211464, + -0.016991777812522046, + -0.004844922012130719, + 0.010125237104633871, + -0.0006617571673224413, + -0.02423516527453047, + -0.013964349950406511, + -0.002420840662100865, + 0.0002485366533294013, + -0.023084525169239933 + ] + }, + "24": { + "top_head": 5, + "top_R": 0.050050267191613346, + "per_head": [ + 0.0008852543161235271, + 0.021516871819612062, + -0.009138629668545898, + -0.008303075932337829, + 0.012235145476287916, + 0.050050267191613346, + -0.008732376604644513, + -0.012716547307187239, + 0.002576200231005693, + -0.004307708316967207, + -0.019522285183458304, + 0.010061023331861884, + 0.007049417552930979, + 0.0027477871841884082, + 0.044039574504288095, + 0.0024504735992134747, + 0.005685471276692106, + 0.00031402951255964307, + 0.0006793394870351967, + 0.0002181863167802908 + ] + }, + "25": { + "top_head": 4, + "top_R": 0.06020661452424007, + "per_head": [ + -0.024193195044325506, + -0.010671325690029779, + -0.0013381953604193445, + 0.005897602527022623, + 0.06020661452424007, + 0.014527303134404417, + 0.02065137595607349, + -0.012663709878624925, + -0.046780057023669684, + 0.006424133029543689, + 0.025522093711375107, + -0.0005934135320710485, + 0.0021422948848589104, + 0.0047871872473510034, + -0.004617123420132598, + 0.006315300971410217, + -0.0012378320005630192, + -0.01473426702251353, + 0.0033234840091290506, + 0.012533357341527001 + ] + }, + "26": { + "top_head": 10, + "top_R": 0.08242937420179375, + "per_head": [ + -0.009253817072406556, + -0.0012105988306141515, + -0.006269260201747059, + -0.006841658722166846, + -0.003959710975619098, + -0.0023335817948334442, + 0.020165338455254575, + 0.005679334952391001, + -0.026168654641640976, + -0.006325568107612662, + 0.08242937420179375, + 0.0502946123505352, + 0.04584784048933921, + 0.028219194483755896, + -0.0040434391297670255, + -0.0038467902591241537, + -0.003447735180757004, + 0.004307525944967943, + -0.015796151970064336, + 0.0045441022722234456 + ] + }, + "27": { + "top_head": 5, + "top_R": 0.10554927295816059, + "per_head": [ + -0.004928663384639123, + -0.0009260684464156908, + 0.004896857481571466, + 0.04508823875232697, + -0.004429143238242332, + 0.10554927295816059, + 0.007325029915908701, + 0.004798999261253804, + -0.0005989735633548239, + 0.0041544229434214396, + 0.05772803954347377, + 0.00868832465994504, + 0.0036004131335843647, + -0.002122681282522863, + -0.015307353338176356, + 0.008540857005266175, + 0.015609116561321888, + 0.012366264862508816, + 0.000642768081016579, + 0.039207340921588923 + ] + }, + "28": { + "top_head": 7, + "top_R": 0.04344987167310966, + "per_head": [ + 0.0020271732956726467, + -0.0030698900870179315, + -0.0035581365842041073, + -0.00893913089593138, + 0.01890297642156609, + 0.0020060768472459833, + -0.003025066195656319, + 0.04344987167310966, + 0.009114221143592881, + -0.003929284876915595, + 0.0005746845818707819, + -0.0020465353573950037, + 0.013405278000484014, + 0.0007283502257467175, + -0.02313991882347066, + -0.00785336096465147, + 0.013174167174064302, + 0.02833920913970194, + 0.009347077739723701, + -0.016827772775230662 + ] + }, + "29": { + "top_head": 4, + "top_R": 0.05788074313123995, + "per_head": [ + 0.0006583682555183891, + -0.010659840809871323, + -0.00760570579277298, + -0.007166254349327647, + 0.05788074313123995, + -0.0028388159466482313, + 0.03323945339814239, + 0.00023790630596653617, + -0.0016789330217736366, + 0.005566385540787174, + -0.011593325397984454, + 0.00012693501697462784, + -0.0052495853424805354, + -0.0001214190770435006, + -0.008300949235705405, + 0.005781335447203732, + 0.012210697800788976, + -0.0009492296092764867, + -0.01274835966462409, + -0.0034564456110690368 + ] + }, + "30": { + "top_head": 19, + "top_R": 0.05614767511181293, + "per_head": [ + 0.008689560986910459, + 0.0006236555372798104, + 0.0024674913206588138, + 0.014136069762919463, + -0.00018910199534941138, + 0.015728739274268797, + 0.004305280190829549, + -0.0016840241859316695, + 0.002315347800672893, + -0.011318182190661425, + -0.0032914744612618733, + -0.004697394549137992, + -0.008761179987624811, + -0.0017935324358357822, + 0.004622968465028163, + -0.007683186520120099, + 0.0068293298671813695, + 0.003787489772402743, + 0.0038747058788893623, + 0.05614767511181293 + ] + }, + "31": { + "top_head": 2, + "top_R": 0.08264338381290322, + "per_head": [ + 0.0005978890060906091, + 0.006387300568594515, + 0.08264338381290322, + 0.019959766600075337, + -0.01795513555517919, + -0.0048787135277967575, + -0.006921832985989966, + 0.00966178953341835, + -0.00626567113164959, + -0.0225686455915563, + 0.013722326384865667, + -0.0021761941825296236, + -0.000758208634952818, + 0.013417226259867075, + 0.015962416131708568, + -0.0011242556437812438, + 0.009620825318704299, + 0.01744185044761553, + 0.04204499934558952, + -0.011898693767065554 + ] + }, + "32": { + "top_head": 5, + "top_R": 0.07447903043432628, + "per_head": [ + -0.004955536806871526, + 0.009420486673214226, + 0.015966848314120256, + -0.003810604520496753, + 0.03701845705858334, + 0.07447903043432628, + -0.010096120546969421, + -0.004524966871906906, + -0.026979142916085073, + -0.0008730748147393633, + 0.013742700890831544, + 0.011823511560561363, + 0.012785011141264865, + -0.008336916659661332, + -0.004708330328498493, + -0.0008219581106703905, + 0.02000343511115649, + 0.0003760428523318079, + 0.02900020357010437, + -0.008145703878699588 + ] + }, + "33": { + "top_head": 12, + "top_R": 0.023452344471991432, + "per_head": [ + -0.0027014760170596345, + -0.003883317483111803, + 0.005563778721309287, + -0.028049418857555235, + 0.020761589081275345, + -0.001783668761694085, + 0.005192200683497365, + -0.01593003836363109, + 0.004869322332706283, + -0.010395914855515968, + -0.00034846510463167583, + 0.019189931782176112, + 0.023452344471991432, + -0.0008954856803347184, + -0.015065360978749316, + 0.005941110305233834, + 0.006748258156652932, + -0.0008439442524162444, + 0.0019679637601425558, + -0.0012240415701286703 + ] + }, + "34": { + "top_head": 4, + "top_R": 0.030279022120833388, + "per_head": [ + -0.010342740014519132, + 0.0007578434589704537, + -0.0009429298253037832, + 0.018682523725480252, + 0.030279022120833388, + -0.0180565433868319, + -0.008948553628467307, + -0.0013383974427371697, + -0.008576944566073576, + 0.008941695926131216, + 0.0046089204994549104, + 0.005085829735551713, + -0.002305296984441929, + -0.0012001754896556584, + 0.005629833115608552, + 0.013762321845950033, + 0.006165380845668918, + 0.0050475607221486885, + -0.0008974492602818036, + -0.010240829932160751 + ] + }, + "35": { + "top_head": 17, + "top_R": 0.045141556754060286, + "per_head": [ + 0.023320029631584235, + -0.019556258255729705, + -0.016981603889820618, + -0.007784225312570819, + -0.002006816598813663, + -0.006883139481267061, + 0.002626040762994337, + 0.00470262336803242, + 0.0029746774170483687, + 0.0008230929972932282, + 0.007540530247723191, + -0.006089729354042471, + -0.003479402083298529, + -0.011484419440456261, + -0.0027402455696341954, + 0.023625091909264272, + -0.01117509118279306, + 0.045141556754060286, + 0.007946822058321153, + 0.009741510602046071 + ] + } + }, + "head_depth": { + "0": { + "peak_layer": 23, + "peak_R": 0.046698124146367204, + "depth_norm": 0.6388888888888888 + }, + "1": { + "peak_layer": 6, + "peak_R": 0.07923933405753308, + "depth_norm": 0.16666666666666666 + }, + "2": { + "peak_layer": 22, + "peak_R": 0.24964245586518252, + "depth_norm": 0.6111111111111112 + }, + "3": { + "peak_layer": 19, + "peak_R": 0.12646392646285057, + "depth_norm": 0.5277777777777778 + }, + "4": { + "peak_layer": 25, + "peak_R": 0.06020661452424007, + "depth_norm": 0.6944444444444444 + }, + "5": { + "peak_layer": 27, + "peak_R": 0.10554927295816059, + "depth_norm": 0.75 + }, + "6": { + "peak_layer": 4, + "peak_R": 0.0340502664948661, + "depth_norm": 0.1111111111111111 + }, + "7": { + "peak_layer": 14, + "peak_R": 0.04948693632875997, + "depth_norm": 0.3888888888888889 + }, + "8": { + "peak_layer": 18, + "peak_R": 0.042658492151948266, + "depth_norm": 0.5 + }, + "9": { + "peak_layer": 2, + "peak_R": 0.023256043949162888, + "depth_norm": 0.05555555555555555 + }, + "10": { + "peak_layer": 26, + "peak_R": 0.08242937420179375, + "depth_norm": 0.7222222222222222 + }, + "11": { + "peak_layer": 17, + "peak_R": 0.08108031504115559, + "depth_norm": 0.4722222222222222 + }, + "12": { + "peak_layer": 6, + "peak_R": 0.13698115120069004, + "depth_norm": 0.16666666666666666 + }, + "13": { + "peak_layer": 12, + "peak_R": 0.10041905713738701, + "depth_norm": 0.3333333333333333 + }, + "14": { + "peak_layer": 12, + "peak_R": 0.10895724721023614, + "depth_norm": 0.3333333333333333 + }, + "15": { + "peak_layer": 13, + "peak_R": 0.07280113988176408, + "depth_norm": 0.3611111111111111 + }, + "16": { + "peak_layer": 6, + "peak_R": 0.029182917737123597, + "depth_norm": 0.16666666666666666 + }, + "17": { + "peak_layer": 35, + "peak_R": 0.045141556754060286, + "depth_norm": 0.9722222222222222 + }, + "18": { + "peak_layer": 31, + "peak_R": 0.04204499934558952, + "depth_norm": 0.8611111111111112 + }, + "19": { + "peak_layer": 30, + "peak_R": 0.05614767511181293, + "depth_norm": 0.8333333333333334 + } + }, + "pre_lcrit_heads": [ + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15, + 16, + 18, + 19 + ], + "post_lcrit_heads": [ + 17 + ] +} \ No newline at end of file diff --git a/data/mi2b_depth/gpt2-medium.json b/data/mi2b_depth/gpt2-medium.json new file mode 100644 index 0000000000000000000000000000000000000000..fa8e60648a7ba96bf1dc0174fadf72f82e0a1d99 --- /dev/null +++ b/data/mi2b_depth/gpt2-medium.json @@ -0,0 +1,1074 @@ +{ + "model": "gpt2-medium", + "N": 24, + "L_crit": 23, + "N_sem": 1, + "n_heads": 16, + "n_pairs_signal": 10, + "R_mean_matrix": [ + [ + -0.0017271260389621942, + -0.0029138854550802683, + 0.004382927984816358, + -0.00546885920474242, + -0.003625582586138206, + 0.00013914906414815533, + 0.0003303102774878703, + 0.001993907817646307, + -0.02019326365911626, + -0.0027665305050904574, + -0.00048412570050898856, + 0.009101658802389034, + 0.0008046255451120277, + 0.015228601490751833, + -0.006507120403988917, + 0.0036203980971821594 + ], + [ + -0.01547522847870316, + 0.03200179411843945, + 0.0015214559710505777, + -0.0025188532962015743, + -0.004600929768546808, + -0.02197510961953622, + -0.02625097779170401, + 0.09370741961917087, + 0.3365492833364765, + -0.006978664113350184, + 0.03594902323563709, + 0.03290424465856519, + -0.008450020630628843, + -0.012116637613204284, + 0.009794210839019355, + 0.06477979128263296 + ], + [ + 0.023500685438056425, + -0.004173633040374428, + 0.04431953895924608, + -0.011813102287548822, + -0.023663994666910305, + -0.011005571954390007, + -0.01695935526622381, + -0.00804809223359522, + 0.02914374249248039, + 0.01378211338363403, + 0.01814272323644734, + 0.03177067887631891, + 0.0071933048212576385, + 0.10194269866208276, + 0.2285219477884361, + -0.017945878517829764 + ], + [ + -0.007350168556472645, + -0.013704163306435294, + 0.008341776706294683, + 0.030152646401736115, + -0.10127635638942871, + -0.0006905584870534904, + 0.06546662665076211, + 0.03550186678603914, + -0.006600965472800905, + 0.016245773142715945, + -0.00713490173095607, + 0.02119443644071997, + -0.000848349349121615, + 0.011810696257986041, + 0.02361498211240428, + 0.017673932623239685 + ], + [ + 0.00012738063959679162, + -0.020904912522335055, + 0.03700599994459579, + -0.02146435132173887, + -0.0006954449918582398, + -0.005682354151585243, + -0.03266470886568654, + 0.005284557125017775, + 0.008316376631282481, + 0.006446049660221981, + 0.020620828803531537, + 0.011367292555144033, + -0.007631844075838184, + 0.042969595931948615, + 0.014703046801298632, + -0.022064610785961402 + ], + [ + 0.00889296693940345, + -0.012718456754098027, + 0.016201867015768794, + -0.016567371161527296, + 0.03425019378319523, + -0.002799084104242947, + 0.0036886552497080527, + -0.0009749562999132041, + 0.03715292272825423, + 0.01696054591039652, + 0.04944222837790258, + -0.04236169095024882, + -0.00281955126253128, + 0.01600337345576618, + 0.08767509058979135, + -0.026175642591338023 + ], + [ + 0.023548094276264954, + -0.04200130781323065, + -0.03799195893326302, + 0.01887600390685055, + 0.02426132571115772, + 0.03866452675958233, + -0.015682043952826734, + -0.04927370672359686, + 0.03599946562726816, + 9.400576543889938e-05, + 0.0006880275032168419, + 0.07643793391784848, + -0.024001312309543145, + 0.009923168988485775, + 0.07186535201937029, + 0.003044628268210799 + ], + [ + 0.3156785948365615, + 0.02747033602165868, + 0.017235200432102925, + -0.019315834849237098, + 0.04155004767795417, + -0.14880297240331292, + -0.05462225461861915, + -0.0028575306498188967, + 0.04811913605043468, + 0.07972518267606257, + -0.004164668818760071, + -0.04267278990347225, + -0.029784616938739655, + 0.009086075360842194, + -0.003930456905217748, + 0.006040678585744485 + ], + [ + -0.01391801292930132, + -0.03079432176722465, + 0.00017929975755360075, + 0.0503151737532232, + -0.025297625272378353, + -0.012978726471267526, + 0.017574680645858202, + -0.012338677786547507, + -0.03411311812661369, + -0.009797388401362187, + 0.07691660152304612, + 0.018511267897435676, + -0.07045896134856874, + -0.01959227151328645, + 0.0423478916959917, + 0.01575616113536039 + ], + [ + -0.015216441435854103, + 0.009784458984743182, + 0.010996889354777353, + 0.01841387387961329, + -0.024100598766969596, + -0.0019057123490974549, + -0.015050453243160785, + -0.15675352781648144, + 0.0014402671367045518, + -0.062341237049692896, + -0.005481024885464722, + -0.019908885314499254, + -0.010954082683621202, + 0.030838263423979502, + 0.008693964243373698, + -0.0011743451774366815 + ], + [ + 0.003924219669284142, + -0.031245418652772465, + -0.0034445966796489954, + 0.02153783476262437, + -0.0008599072820477416, + 0.0056532077475726795, + 0.017449551894770314, + 0.041583090653389145, + 0.0029714963969351936, + 0.007783191416226511, + 0.042263799992732005, + -0.0018918201068566787, + 0.0010182435295711014, + -0.027731524528443286, + 0.03552430085534891, + -0.026729835677174678 + ], + [ + -0.015035570332447392, + -0.02420117919736275, + -0.0007238285709643604, + -0.05052411693469744, + 0.09660101753867371, + -0.008780169747508004, + 0.0622297570414043, + -0.009421034159968146, + 0.019738769599568676, + -0.002541809197227694, + 0.03976907166235754, + 0.0015188942381240118, + 0.03011082719170114, + 0.00047088005854341016, + -0.002558105758724674, + -0.0044371924856969735 + ], + [ + -0.0006611378661195431, + -0.002746194140870629, + -0.003088955854079646, + 0.0028341816928789945, + 0.005318469502803182, + -0.01896217120713613, + 0.002309398660784487, + 0.020934542757461407, + 0.03556150441703269, + -0.0122350268122614, + 0.011474107133597877, + 0.020528385415816065, + 0.0022289894575444795, + -0.016144620532666763, + -0.0025791592575963437, + -0.0009423186050746567 + ], + [ + 0.030745781594002753, + 0.00569772086070246, + -0.002062632224987123, + 0.006882637144628387, + -0.0019297766044722268, + -0.017800659907828125, + 0.01589398786917219, + -0.01691108965751059, + -0.0007746796669491125, + -0.011121172058467145, + 0.02117223341598871, + -0.012824385016065646, + -0.019470714615006122, + 0.00471690186118339, + -0.01771167581156129, + 0.004877724822084839 + ], + [ + -0.018177010514043527, + -0.003952755273578598, + 0.024347062039759934, + 0.003116598600038917, + 0.002337490432539179, + 0.004587324871557119, + 0.021571511255905475, + -0.011267312574189347, + -0.0012621588955419078, + 0.014188740661388238, + 0.008165182607110568, + -0.0036371170885152785, + 0.0027706126979372414, + 0.0022809049342213854, + -0.0050440603734153155, + 0.13012221617188394 + ], + [ + -0.006255092306030682, + -0.01769073109274719, + 0.028920789848476115, + -6.39242473530293e-05, + -0.003257942367879109, + -0.0020625421362029306, + 0.0250737977557677, + 0.0035908679732420553, + 0.0016202426406212052, + 0.0052727773070219116, + 0.114912139577151, + -0.03761831806471219, + 0.060206153864020895, + -0.004599322845619193, + -0.0046010257422464795, + -0.04297556449546085 + ], + [ + -0.001920337791390194, + 0.008095795590666343, + 0.0018824882688178609, + 0.005750167549449333, + 0.0037874154919115013, + -0.0004888229973649444, + 0.002472119849866253, + 0.012631551052654635, + 0.00011096375161011498, + 0.0014663732276074316, + 0.0002541083753358031, + -0.0030582082301271025, + 8.933964555584568e-06, + 0.03308475224025483, + 0.014785601128538226, + 0.0044276841703974265 + ], + [ + 0.00533122178484399, + 0.004777005381321825, + 0.02088795232106163, + -0.005929238233790516, + 0.0062186702027387354, + -0.0025612259493016138, + -0.005503552829615044, + -0.001686891987134022, + 0.019242247190738873, + -0.03318296621342641, + -0.007786894157692316, + 0.03851479281032039, + 0.003529399485638593, + 0.0028986857848597716, + -0.012622178752293215, + -0.006919403827789038 + ], + [ + -0.004886523197844368, + -0.002650618971065923, + -0.007508398470622096, + 0.014343830762353817, + 0.007893257086524112, + -0.005954906985731692, + 0.030425980453225122, + -0.0015341295313802995, + 0.014096747467355513, + 0.0077831108316435875, + 0.00013068960597379932, + 0.0034111769115171297, + -0.002675529927436643, + 0.01559102945120873, + -0.005703239792426845, + 0.03167796233739738 + ], + [ + 0.005080101613508632, + 7.335168985842365e-05, + 0.006224756280246604, + -0.006952447193212863, + -0.036018827051399235, + 0.0009310370835446298, + -0.014615912563189188, + 0.0021368737879675402, + 0.030470641424174282, + 0.011587155424721586, + 0.0041647447207340945, + -0.005016422699282179, + -0.0009657284718819499, + 0.006829139186022774, + -0.0031191954468558656, + 0.020785909340070595 + ], + [ + -0.007469852318719333, + -0.015237288951250283, + 0.06999791827262274, + -0.042746713980715646, + 0.010671300456611877, + 0.003718682637153117, + 0.007799532643614211, + -0.01155915493558495, + 0.016908474812788583, + 0.006819016232484007, + 0.0026245093661991297, + 0.006251182521779185, + -0.018194887506617557, + 0.0072336289643113675, + -0.0009659478767530006, + -0.020652691255009195 + ], + [ + 0.01207012774041991, + 0.05898879429026895, + 0.004276396039267876, + -0.00042236801987644707, + 0.005056950249040809, + 0.0013465612458975042, + -0.010907748973710649, + 0.003043147908856418, + 0.00032080652036172797, + -0.008674613745158728, + -0.004615458439246299, + 0.009046086743760066, + -0.0012863059786400236, + -0.0024978717746230424, + -0.003512753738294595, + 0.005214007430384836 + ], + [ + 0.004247184217483857, + 0.012814098390208311, + 0.014911127155337406, + -0.00870597014757676, + -0.0008020906598705489, + 0.011915123936329229, + -0.0021320260508580114, + -0.004866290542864918, + 0.0005046369833274411, + 0.03888398416126673, + -0.002584878149204956, + 0.010532654151462872, + 0.0003865927136379985, + -0.0009916982858572284, + 0.008333851393087163, + 0.0011979468947074337 + ], + [ + 0.006394395080650056, + 0.028055999061650072, + -0.007963718967468328, + -0.0014438467166025084, + -0.021730758799318614, + 0.04027858712751371, + 0.018256340254386893, + 0.006569320240574196, + -0.0009880406470108931, + -0.008162488941635465, + 0.005096962787476294, + 0.014559432303788663, + 0.007525107934352296, + -0.002292413650608834, + 0.0026643501108688677, + -0.060972899483564505 + ] + ], + "head_profiles": { + "0": { + "top_head": 13, + "top_R": 0.015228601490751833, + "per_head": [ + -0.0017271260389621942, + -0.0029138854550802683, + 0.004382927984816358, + -0.00546885920474242, + -0.003625582586138206, + 0.00013914906414815533, + 0.0003303102774878703, + 0.001993907817646307, + -0.02019326365911626, + -0.0027665305050904574, + -0.00048412570050898856, + 0.009101658802389034, + 0.0008046255451120277, + 0.015228601490751833, + -0.006507120403988917, + 0.0036203980971821594 + ] + }, + "1": { + "top_head": 8, + "top_R": 0.3365492833364765, + "per_head": [ + -0.01547522847870316, + 0.03200179411843945, + 0.0015214559710505777, + -0.0025188532962015743, + -0.004600929768546808, + -0.02197510961953622, + -0.02625097779170401, + 0.09370741961917087, + 0.3365492833364765, + -0.006978664113350184, + 0.03594902323563709, + 0.03290424465856519, + -0.008450020630628843, + -0.012116637613204284, + 0.009794210839019355, + 0.06477979128263296 + ] + }, + "2": { + "top_head": 14, + "top_R": 0.2285219477884361, + "per_head": [ + 0.023500685438056425, + -0.004173633040374428, + 0.04431953895924608, + -0.011813102287548822, + -0.023663994666910305, + -0.011005571954390007, + -0.01695935526622381, + -0.00804809223359522, + 0.02914374249248039, + 0.01378211338363403, + 0.01814272323644734, + 0.03177067887631891, + 0.0071933048212576385, + 0.10194269866208276, + 0.2285219477884361, + -0.017945878517829764 + ] + }, + "3": { + "top_head": 6, + "top_R": 0.06546662665076211, + "per_head": [ + -0.007350168556472645, + -0.013704163306435294, + 0.008341776706294683, + 0.030152646401736115, + -0.10127635638942871, + -0.0006905584870534904, + 0.06546662665076211, + 0.03550186678603914, + -0.006600965472800905, + 0.016245773142715945, + -0.00713490173095607, + 0.02119443644071997, + -0.000848349349121615, + 0.011810696257986041, + 0.02361498211240428, + 0.017673932623239685 + ] + }, + "4": { + "top_head": 13, + "top_R": 0.042969595931948615, + "per_head": [ + 0.00012738063959679162, + -0.020904912522335055, + 0.03700599994459579, + -0.02146435132173887, + -0.0006954449918582398, + -0.005682354151585243, + -0.03266470886568654, + 0.005284557125017775, + 0.008316376631282481, + 0.006446049660221981, + 0.020620828803531537, + 0.011367292555144033, + -0.007631844075838184, + 0.042969595931948615, + 0.014703046801298632, + -0.022064610785961402 + ] + }, + "5": { + "top_head": 14, + "top_R": 0.08767509058979135, + "per_head": [ + 0.00889296693940345, + -0.012718456754098027, + 0.016201867015768794, + -0.016567371161527296, + 0.03425019378319523, + -0.002799084104242947, + 0.0036886552497080527, + -0.0009749562999132041, + 0.03715292272825423, + 0.01696054591039652, + 0.04944222837790258, + -0.04236169095024882, + -0.00281955126253128, + 0.01600337345576618, + 0.08767509058979135, + -0.026175642591338023 + ] + }, + "6": { + "top_head": 11, + "top_R": 0.07643793391784848, + "per_head": [ + 0.023548094276264954, + -0.04200130781323065, + -0.03799195893326302, + 0.01887600390685055, + 0.02426132571115772, + 0.03866452675958233, + -0.015682043952826734, + -0.04927370672359686, + 0.03599946562726816, + 9.400576543889938e-05, + 0.0006880275032168419, + 0.07643793391784848, + -0.024001312309543145, + 0.009923168988485775, + 0.07186535201937029, + 0.003044628268210799 + ] + }, + "7": { + "top_head": 0, + "top_R": 0.3156785948365615, + "per_head": [ + 0.3156785948365615, + 0.02747033602165868, + 0.017235200432102925, + -0.019315834849237098, + 0.04155004767795417, + -0.14880297240331292, + -0.05462225461861915, + -0.0028575306498188967, + 0.04811913605043468, + 0.07972518267606257, + -0.004164668818760071, + -0.04267278990347225, + -0.029784616938739655, + 0.009086075360842194, + -0.003930456905217748, + 0.006040678585744485 + ] + }, + "8": { + "top_head": 10, + "top_R": 0.07691660152304612, + "per_head": [ + -0.01391801292930132, + -0.03079432176722465, + 0.00017929975755360075, + 0.0503151737532232, + -0.025297625272378353, + -0.012978726471267526, + 0.017574680645858202, + -0.012338677786547507, + -0.03411311812661369, + -0.009797388401362187, + 0.07691660152304612, + 0.018511267897435676, + -0.07045896134856874, + -0.01959227151328645, + 0.0423478916959917, + 0.01575616113536039 + ] + }, + "9": { + "top_head": 13, + "top_R": 0.030838263423979502, + "per_head": [ + -0.015216441435854103, + 0.009784458984743182, + 0.010996889354777353, + 0.01841387387961329, + -0.024100598766969596, + -0.0019057123490974549, + -0.015050453243160785, + -0.15675352781648144, + 0.0014402671367045518, + -0.062341237049692896, + -0.005481024885464722, + -0.019908885314499254, + -0.010954082683621202, + 0.030838263423979502, + 0.008693964243373698, + -0.0011743451774366815 + ] + }, + "10": { + "top_head": 10, + "top_R": 0.042263799992732005, + "per_head": [ + 0.003924219669284142, + -0.031245418652772465, + -0.0034445966796489954, + 0.02153783476262437, + -0.0008599072820477416, + 0.0056532077475726795, + 0.017449551894770314, + 0.041583090653389145, + 0.0029714963969351936, + 0.007783191416226511, + 0.042263799992732005, + -0.0018918201068566787, + 0.0010182435295711014, + -0.027731524528443286, + 0.03552430085534891, + -0.026729835677174678 + ] + }, + "11": { + "top_head": 4, + "top_R": 0.09660101753867371, + "per_head": [ + -0.015035570332447392, + -0.02420117919736275, + -0.0007238285709643604, + -0.05052411693469744, + 0.09660101753867371, + -0.008780169747508004, + 0.0622297570414043, + -0.009421034159968146, + 0.019738769599568676, + -0.002541809197227694, + 0.03976907166235754, + 0.0015188942381240118, + 0.03011082719170114, + 0.00047088005854341016, + -0.002558105758724674, + -0.0044371924856969735 + ] + }, + "12": { + "top_head": 8, + "top_R": 0.03556150441703269, + "per_head": [ + -0.0006611378661195431, + -0.002746194140870629, + -0.003088955854079646, + 0.0028341816928789945, + 0.005318469502803182, + -0.01896217120713613, + 0.002309398660784487, + 0.020934542757461407, + 0.03556150441703269, + -0.0122350268122614, + 0.011474107133597877, + 0.020528385415816065, + 0.0022289894575444795, + -0.016144620532666763, + -0.0025791592575963437, + -0.0009423186050746567 + ] + }, + "13": { + "top_head": 0, + "top_R": 0.030745781594002753, + "per_head": [ + 0.030745781594002753, + 0.00569772086070246, + -0.002062632224987123, + 0.006882637144628387, + -0.0019297766044722268, + -0.017800659907828125, + 0.01589398786917219, + -0.01691108965751059, + -0.0007746796669491125, + -0.011121172058467145, + 0.02117223341598871, + -0.012824385016065646, + -0.019470714615006122, + 0.00471690186118339, + -0.01771167581156129, + 0.004877724822084839 + ] + }, + "14": { + "top_head": 15, + "top_R": 0.13012221617188394, + "per_head": [ + -0.018177010514043527, + -0.003952755273578598, + 0.024347062039759934, + 0.003116598600038917, + 0.002337490432539179, + 0.004587324871557119, + 0.021571511255905475, + -0.011267312574189347, + -0.0012621588955419078, + 0.014188740661388238, + 0.008165182607110568, + -0.0036371170885152785, + 0.0027706126979372414, + 0.0022809049342213854, + -0.0050440603734153155, + 0.13012221617188394 + ] + }, + "15": { + "top_head": 10, + "top_R": 0.114912139577151, + "per_head": [ + -0.006255092306030682, + -0.01769073109274719, + 0.028920789848476115, + -6.39242473530293e-05, + -0.003257942367879109, + -0.0020625421362029306, + 0.0250737977557677, + 0.0035908679732420553, + 0.0016202426406212052, + 0.0052727773070219116, + 0.114912139577151, + -0.03761831806471219, + 0.060206153864020895, + -0.004599322845619193, + -0.0046010257422464795, + -0.04297556449546085 + ] + }, + "16": { + "top_head": 13, + "top_R": 0.03308475224025483, + "per_head": [ + -0.001920337791390194, + 0.008095795590666343, + 0.0018824882688178609, + 0.005750167549449333, + 0.0037874154919115013, + -0.0004888229973649444, + 0.002472119849866253, + 0.012631551052654635, + 0.00011096375161011498, + 0.0014663732276074316, + 0.0002541083753358031, + -0.0030582082301271025, + 8.933964555584568e-06, + 0.03308475224025483, + 0.014785601128538226, + 0.0044276841703974265 + ] + }, + "17": { + "top_head": 11, + "top_R": 0.03851479281032039, + "per_head": [ + 0.00533122178484399, + 0.004777005381321825, + 0.02088795232106163, + -0.005929238233790516, + 0.0062186702027387354, + -0.0025612259493016138, + -0.005503552829615044, + -0.001686891987134022, + 0.019242247190738873, + -0.03318296621342641, + -0.007786894157692316, + 0.03851479281032039, + 0.003529399485638593, + 0.0028986857848597716, + -0.012622178752293215, + -0.006919403827789038 + ] + }, + "18": { + "top_head": 15, + "top_R": 0.03167796233739738, + "per_head": [ + -0.004886523197844368, + -0.002650618971065923, + -0.007508398470622096, + 0.014343830762353817, + 0.007893257086524112, + -0.005954906985731692, + 0.030425980453225122, + -0.0015341295313802995, + 0.014096747467355513, + 0.0077831108316435875, + 0.00013068960597379932, + 0.0034111769115171297, + -0.002675529927436643, + 0.01559102945120873, + -0.005703239792426845, + 0.03167796233739738 + ] + }, + "19": { + "top_head": 8, + "top_R": 0.030470641424174282, + "per_head": [ + 0.005080101613508632, + 7.335168985842365e-05, + 0.006224756280246604, + -0.006952447193212863, + -0.036018827051399235, + 0.0009310370835446298, + -0.014615912563189188, + 0.0021368737879675402, + 0.030470641424174282, + 0.011587155424721586, + 0.0041647447207340945, + -0.005016422699282179, + -0.0009657284718819499, + 0.006829139186022774, + -0.0031191954468558656, + 0.020785909340070595 + ] + }, + "20": { + "top_head": 2, + "top_R": 0.06999791827262274, + "per_head": [ + -0.007469852318719333, + -0.015237288951250283, + 0.06999791827262274, + -0.042746713980715646, + 0.010671300456611877, + 0.003718682637153117, + 0.007799532643614211, + -0.01155915493558495, + 0.016908474812788583, + 0.006819016232484007, + 0.0026245093661991297, + 0.006251182521779185, + -0.018194887506617557, + 0.0072336289643113675, + -0.0009659478767530006, + -0.020652691255009195 + ] + }, + "21": { + "top_head": 1, + "top_R": 0.05898879429026895, + "per_head": [ + 0.01207012774041991, + 0.05898879429026895, + 0.004276396039267876, + -0.00042236801987644707, + 0.005056950249040809, + 0.0013465612458975042, + -0.010907748973710649, + 0.003043147908856418, + 0.00032080652036172797, + -0.008674613745158728, + -0.004615458439246299, + 0.009046086743760066, + -0.0012863059786400236, + -0.0024978717746230424, + -0.003512753738294595, + 0.005214007430384836 + ] + }, + "22": { + "top_head": 9, + "top_R": 0.03888398416126673, + "per_head": [ + 0.004247184217483857, + 0.012814098390208311, + 0.014911127155337406, + -0.00870597014757676, + -0.0008020906598705489, + 0.011915123936329229, + -0.0021320260508580114, + -0.004866290542864918, + 0.0005046369833274411, + 0.03888398416126673, + -0.002584878149204956, + 0.010532654151462872, + 0.0003865927136379985, + -0.0009916982858572284, + 0.008333851393087163, + 0.0011979468947074337 + ] + }, + "23": { + "top_head": 5, + "top_R": 0.04027858712751371, + "per_head": [ + 0.006394395080650056, + 0.028055999061650072, + -0.007963718967468328, + -0.0014438467166025084, + -0.021730758799318614, + 0.04027858712751371, + 0.018256340254386893, + 0.006569320240574196, + -0.0009880406470108931, + -0.008162488941635465, + 0.005096962787476294, + 0.014559432303788663, + 0.007525107934352296, + -0.002292413650608834, + 0.0026643501108688677, + -0.060972899483564505 + ] + } + }, + "head_depth": { + "0": { + "peak_layer": 7, + "peak_R": 0.3156785948365615, + "depth_norm": 0.2916666666666667 + }, + "1": { + "peak_layer": 21, + "peak_R": 0.05898879429026895, + "depth_norm": 0.875 + }, + "2": { + "peak_layer": 20, + "peak_R": 0.06999791827262274, + "depth_norm": 0.8333333333333334 + }, + "3": { + "peak_layer": 8, + "peak_R": 0.0503151737532232, + "depth_norm": 0.3333333333333333 + }, + "4": { + "peak_layer": 11, + "peak_R": 0.09660101753867371, + "depth_norm": 0.4583333333333333 + }, + "5": { + "peak_layer": 23, + "peak_R": 0.04027858712751371, + "depth_norm": 0.9583333333333334 + }, + "6": { + "peak_layer": 3, + "peak_R": 0.06546662665076211, + "depth_norm": 0.125 + }, + "7": { + "peak_layer": 1, + "peak_R": 0.09370741961917087, + "depth_norm": 0.041666666666666664 + }, + "8": { + "peak_layer": 1, + "peak_R": 0.3365492833364765, + "depth_norm": 0.041666666666666664 + }, + "9": { + "peak_layer": 7, + "peak_R": 0.07972518267606257, + "depth_norm": 0.2916666666666667 + }, + "10": { + "peak_layer": 15, + "peak_R": 0.114912139577151, + "depth_norm": 0.625 + }, + "11": { + "peak_layer": 6, + "peak_R": 0.07643793391784848, + "depth_norm": 0.25 + }, + "12": { + "peak_layer": 15, + "peak_R": 0.060206153864020895, + "depth_norm": 0.625 + }, + "13": { + "peak_layer": 2, + "peak_R": 0.10194269866208276, + "depth_norm": 0.08333333333333333 + }, + "14": { + "peak_layer": 2, + "peak_R": 0.2285219477884361, + "depth_norm": 0.08333333333333333 + }, + "15": { + "peak_layer": 14, + "peak_R": 0.13012221617188394, + "depth_norm": 0.5833333333333334 + } + }, + "pre_lcrit_heads": [ + 0, + 1, + 2, + 3, + 4, + 6, + 7, + 8, + 9, + 10, + 11, + 12, + 13, + 14, + 15 + ], + "post_lcrit_heads": [ + 5 + ] +} \ No newline at end of file diff --git a/data/mi2b_depth/gpt2.json b/data/mi2b_depth/gpt2.json new file mode 100644 index 0000000000000000000000000000000000000000..6e8bf53f9f20abac477aeacdf4b4d35281d0ba93 --- /dev/null +++ b/data/mi2b_depth/gpt2.json @@ -0,0 +1,474 @@ +{ + "model": "gpt2", + "N": 12, + "L_crit": 11, + "N_sem": 1, + "n_heads": 12, + "n_pairs_signal": 9, + "R_mean_matrix": [ + [ + -0.04715073039943866, + -0.029001056429040186, + 0.030221303599445668, + -0.2290398295024203, + 0.09090531049402659, + 0.0017913200014473443, + 0.003260929010006637, + 0.15089775380984966, + -0.07491029845981216, + -0.1572386702988569, + 0.12717930506710762, + -0.10473274400459726 + ], + [ + -0.1318337397727064, + -0.4213162623472477, + 0.23165700066516834, + -0.13368952700599007, + -0.14882521513543304, + 0.06787011883760985, + 0.03787061894591053, + -0.05332101729179471, + -0.11831851365089986, + -0.017984037081001155, + -0.06907344970072377, + 0.07060005397101085 + ], + [ + -0.11489626119613244, + 0.04623448790789987, + 0.026962173889546106, + -0.11237564004313125, + 0.09365250116595426, + 0.10423854373191248, + 0.0316296186254476, + 0.008260330808955937, + 0.04066127607085644, + -0.19282725746184776, + -0.6512256171106814, + 0.09012197026704417 + ], + [ + 0.008846466762636764, + 0.05728363146232206, + 0.09741414023451352, + 0.04242838736813755, + 0.07451253629992917, + -0.013316969840191316, + 0.07557048055595837, + -0.14088269560099778, + 0.060783744078698305, + 0.024776118868615395, + 0.04228801195933449, + -0.06361597468728013 + ], + [ + -0.04014155411246823, + 0.04963818002405023, + -0.008388670540789983, + 0.054971933612966185, + 0.03649678935095188, + -0.0213560104046007, + -0.0047868937118550345, + -0.15540449805590698, + -0.4844041040178607, + -0.03300573434805098, + -0.06122562114925001, + 0.0888176702208909 + ], + [ + 0.08127930788901983, + 0.06415212565542718, + 0.03320935833541057, + 0.04739663649971762, + 0.0038670206737053186, + 0.1488186179416576, + 0.08010108171716371, + 0.00638253806427289, + -0.07033085979384841, + -0.05390065189145713, + 0.007852939295914102, + -0.010056956073631725 + ], + [ + 0.01796811593864487, + 0.006407069296773457, + -0.04709849150414727, + -0.013469603267306889, + 0.08042992587751102, + -0.11116491130235996, + -0.05504102048706353, + 0.019918679485582954, + -0.025103327371914164, + 0.15315659907572943, + 0.08229150363759306, + -0.08117270225550653 + ], + [ + -0.10180967152804508, + 0.03222429996409318, + 0.034971915990998886, + 0.01305773491803531, + 0.015126826029587778, + 0.020191478612709885, + -0.18508767821386374, + 0.14287022030667418, + 0.05501531002267371, + 0.009831980393353819, + 0.026352788818261988, + 0.1212155730197087 + ], + [ + -0.09801482577065705, + 0.2007654816930569, + 0.15248623335356487, + -0.004064701811150894, + -0.004911520154477286, + 0.06309196023033957, + 0.11031133439239438, + 0.03070863819006671, + 0.008041175382738443, + 0.05176201773231278, + 0.004253190095890178, + -0.054719526908106234 + ], + [ + 0.004145712404501323, + 0.007987218222317951, + -0.0114119612965377, + 0.13098756915939333, + 0.05301174024499756, + -0.015788334659833432, + 0.01729988009627639, + -0.03479224432470708, + -0.014176787584326812, + -0.004649484734515968, + 0.037812927152647766, + 0.160472915680756 + ], + [ + -0.007913238026429558, + -0.011491038533055666, + 0.03442674451579263, + 0.018874895421248167, + 0.01997252411042421, + 0.07447600483803454, + 0.01995936444025323, + -0.03658935480968632, + 0.17273636446835489, + -0.06751505704989773, + -0.0020313445353767642, + 0.06330243710629628 + ], + [ + 0.27824050979385456, + 0.017357301641577814, + 0.0006097296950808134, + 0.007686845096684596, + 0.030463161811448893, + -0.06787734537202234, + 0.0018237090926640316, + -0.024491175126714396, + 0.017871973304610638, + -0.06529732346562883, + -0.007579777300539466, + 0.030240790304848806 + ] + ], + "head_profiles": { + "0": { + "top_head": 7, + "top_R": 0.15089775380984966, + "per_head": [ + -0.04715073039943866, + -0.029001056429040186, + 0.030221303599445668, + -0.2290398295024203, + 0.09090531049402659, + 0.0017913200014473443, + 0.003260929010006637, + 0.15089775380984966, + -0.07491029845981216, + -0.1572386702988569, + 0.12717930506710762, + -0.10473274400459726 + ] + }, + "1": { + "top_head": 2, + "top_R": 0.23165700066516834, + "per_head": [ + -0.1318337397727064, + -0.4213162623472477, + 0.23165700066516834, + -0.13368952700599007, + -0.14882521513543304, + 0.06787011883760985, + 0.03787061894591053, + -0.05332101729179471, + -0.11831851365089986, + -0.017984037081001155, + -0.06907344970072377, + 0.07060005397101085 + ] + }, + "2": { + "top_head": 5, + "top_R": 0.10423854373191248, + "per_head": [ + -0.11489626119613244, + 0.04623448790789987, + 0.026962173889546106, + -0.11237564004313125, + 0.09365250116595426, + 0.10423854373191248, + 0.0316296186254476, + 0.008260330808955937, + 0.04066127607085644, + -0.19282725746184776, + -0.6512256171106814, + 0.09012197026704417 + ] + }, + "3": { + "top_head": 2, + "top_R": 0.09741414023451352, + "per_head": [ + 0.008846466762636764, + 0.05728363146232206, + 0.09741414023451352, + 0.04242838736813755, + 0.07451253629992917, + -0.013316969840191316, + 0.07557048055595837, + -0.14088269560099778, + 0.060783744078698305, + 0.024776118868615395, + 0.04228801195933449, + -0.06361597468728013 + ] + }, + "4": { + "top_head": 11, + "top_R": 0.0888176702208909, + "per_head": [ + -0.04014155411246823, + 0.04963818002405023, + -0.008388670540789983, + 0.054971933612966185, + 0.03649678935095188, + -0.0213560104046007, + -0.0047868937118550345, + -0.15540449805590698, + -0.4844041040178607, + -0.03300573434805098, + -0.06122562114925001, + 0.0888176702208909 + ] + }, + "5": { + "top_head": 5, + "top_R": 0.1488186179416576, + "per_head": [ + 0.08127930788901983, + 0.06415212565542718, + 0.03320935833541057, + 0.04739663649971762, + 0.0038670206737053186, + 0.1488186179416576, + 0.08010108171716371, + 0.00638253806427289, + -0.07033085979384841, + -0.05390065189145713, + 0.007852939295914102, + -0.010056956073631725 + ] + }, + "6": { + "top_head": 9, + "top_R": 0.15315659907572943, + "per_head": [ + 0.01796811593864487, + 0.006407069296773457, + -0.04709849150414727, + -0.013469603267306889, + 0.08042992587751102, + -0.11116491130235996, + -0.05504102048706353, + 0.019918679485582954, + -0.025103327371914164, + 0.15315659907572943, + 0.08229150363759306, + -0.08117270225550653 + ] + }, + "7": { + "top_head": 7, + "top_R": 0.14287022030667418, + "per_head": [ + -0.10180967152804508, + 0.03222429996409318, + 0.034971915990998886, + 0.01305773491803531, + 0.015126826029587778, + 0.020191478612709885, + -0.18508767821386374, + 0.14287022030667418, + 0.05501531002267371, + 0.009831980393353819, + 0.026352788818261988, + 0.1212155730197087 + ] + }, + "8": { + "top_head": 1, + "top_R": 0.2007654816930569, + "per_head": [ + -0.09801482577065705, + 0.2007654816930569, + 0.15248623335356487, + -0.004064701811150894, + -0.004911520154477286, + 0.06309196023033957, + 0.11031133439239438, + 0.03070863819006671, + 0.008041175382738443, + 0.05176201773231278, + 0.004253190095890178, + -0.054719526908106234 + ] + }, + "9": { + "top_head": 11, + "top_R": 0.160472915680756, + "per_head": [ + 0.004145712404501323, + 0.007987218222317951, + -0.0114119612965377, + 0.13098756915939333, + 0.05301174024499756, + -0.015788334659833432, + 0.01729988009627639, + -0.03479224432470708, + -0.014176787584326812, + -0.004649484734515968, + 0.037812927152647766, + 0.160472915680756 + ] + }, + "10": { + "top_head": 8, + "top_R": 0.17273636446835489, + "per_head": [ + -0.007913238026429558, + -0.011491038533055666, + 0.03442674451579263, + 0.018874895421248167, + 0.01997252411042421, + 0.07447600483803454, + 0.01995936444025323, + -0.03658935480968632, + 0.17273636446835489, + -0.06751505704989773, + -0.0020313445353767642, + 0.06330243710629628 + ] + }, + "11": { + "top_head": 0, + "top_R": 0.27824050979385456, + "per_head": [ + 0.27824050979385456, + 0.017357301641577814, + 0.0006097296950808134, + 0.007686845096684596, + 0.030463161811448893, + -0.06787734537202234, + 0.0018237090926640316, + -0.024491175126714396, + 0.017871973304610638, + -0.06529732346562883, + -0.007579777300539466, + 0.030240790304848806 + ] + } + }, + "head_depth": { + "0": { + "peak_layer": 11, + "peak_R": 0.27824050979385456, + "depth_norm": 0.9166666666666666 + }, + "1": { + "peak_layer": 8, + "peak_R": 0.2007654816930569, + "depth_norm": 0.6666666666666666 + }, + "2": { + "peak_layer": 1, + "peak_R": 0.23165700066516834, + "depth_norm": 0.08333333333333333 + }, + "3": { + "peak_layer": 9, + "peak_R": 0.13098756915939333, + "depth_norm": 0.75 + }, + "4": { + "peak_layer": 2, + "peak_R": 0.09365250116595426, + "depth_norm": 0.16666666666666666 + }, + "5": { + "peak_layer": 5, + "peak_R": 0.1488186179416576, + "depth_norm": 0.4166666666666667 + }, + "6": { + "peak_layer": 8, + "peak_R": 0.11031133439239438, + "depth_norm": 0.6666666666666666 + }, + "7": { + "peak_layer": 0, + "peak_R": 0.15089775380984966, + "depth_norm": 0.0 + }, + "8": { + "peak_layer": 10, + "peak_R": 0.17273636446835489, + "depth_norm": 0.8333333333333334 + }, + "9": { + "peak_layer": 6, + "peak_R": 0.15315659907572943, + "depth_norm": 0.5 + }, + "10": { + "peak_layer": 0, + "peak_R": 0.12717930506710762, + "depth_norm": 0.0 + }, + "11": { + "peak_layer": 9, + "peak_R": 0.160472915680756, + "depth_norm": 0.75 + } + }, + "pre_lcrit_heads": [ + 1, + 2, + 3, + 4, + 5, + 6, + 7, + 8, + 9, + 10, + 11 + ], + "post_lcrit_heads": [ + 0 + ] +} \ No newline at end of file diff --git a/data/mi3_periodicity/gpt2-large.json b/data/mi3_periodicity/gpt2-large.json new file mode 100644 index 0000000000000000000000000000000000000000..173735a6a38909ac82b053fe6fb4474c2662b017 --- /dev/null +++ b/data/mi3_periodicity/gpt2-large.json @@ -0,0 +1,178 @@ +{ + "model": "gpt2-large", + "N": 36, + "L_crit": 33, + "N_sem": 3, + "sr_curve": [ + 14.47738881363044, + 5.227110739349127, + 5.187858631474805, + 6.143202398608626, + 6.860827759994744, + 6.303258693247651, + 6.623308729794722, + 7.998942062348251, + 7.563764749229092, + 10.016432342067255, + 9.623633479897531, + 9.96236466710416, + 10.99048786394631, + 10.368821163488269, + 9.819520090021111, + 10.586574614857517, + 11.031872590984003, + 10.076315839602705, + 9.660302285233902, + 9.731558322224434, + 10.745459704783675, + 9.195562352441039, + 8.565956270738315, + 8.78153713784886, + 8.924758186942997, + 10.284102530081402, + 8.251956010676817, + 10.011453461836332, + 9.414813804942495, + 11.874396360053556, + 9.664489671673811, + 10.949332121596978, + 9.526734958479397, + 10.49278153341553, + 8.503772422969302, + 9.76867953584213 + ], + "frob_curve": [ + 2.2478068709373473, + 1.9068263083696366, + 2.8963742703199387, + 3.1239175885915755, + 3.4280447721481324, + 4.15092386007309, + 4.229616641998291, + 4.29519567489624, + 4.333398985862732, + 4.67714604139328, + 4.476090490818024, + 4.468318152427673, + 4.837147891521454, + 4.655142223834991, + 4.621856284141541, + 4.794538426399231, + 5.041932606697083, + 4.606095886230468, + 4.511252284049988, + 4.702972209453582, + 4.621228837966919, + 4.115156149864196, + 3.877094030380249, + 3.8796911239624023, + 3.796297585964203, + 3.6912230849266052, + 3.800480902194977, + 3.802728271484375, + 3.603396499156952, + 3.660360038280487, + 3.77968407869339, + 4.0416216373443605, + 4.24884033203125, + 4.949480199813843, + 5.8888376474380495, + 7.717085576057434 + ], + "spec_curve": [ + 0.562838825583458, + 0.8952949643135071, + 1.3111250072717666, + 1.3094798356294632, + 1.3785526663064958, + 1.6834433734416963, + 1.7444728761911392, + 1.7746364891529083, + 1.7225086748600007, + 1.542852258682251, + 1.498388135433197, + 1.4475996315479278, + 1.534312254190445, + 1.4718647181987763, + 1.524321973323822, + 1.5310561656951904, + 1.5522982776165009, + 1.4716392427682876, + 1.4891833782196044, + 1.5549124360084534, + 1.4739458441734314, + 1.4183288604021071, + 1.4203574538230896, + 1.38409683406353, + 1.3155996054410934, + 1.1837209939956665, + 1.4324533283710479, + 1.2518831133842467, + 1.2062453985214234, + 1.133369255065918, + 1.2574486911296845, + 1.2546493262052536, + 1.4197751104831695, + 1.5721863448619842, + 2.104900848865509, + 2.591031068563461 + ], + "periodicity_sr": { + "k_dominant_ac": 2.0, + "k_nsem": 3.0, + "k_nsem_power": 35.480487506965794, + "k_nsem_rank_in_ac": 15.0, + "k_nsem_ratio": 0.02024593043280426, + "total_ac_power": 1752.4750282396085, + "power_spectrum": [ + 1.00729648987671e-26, + 196.75000840920433, + 402.12213645053623, + 35.480487506965794, + 80.64565195673701, + 45.1922946567768, + 72.8681435453787, + 128.80818410484346, + 22.886822573283073, + 221.11157691675885, + 75.74511638930025, + 140.30393695332066, + 97.11484876314611, + 49.37034419786297, + 51.45599837383223, + 85.99586774384623, + 201.93628799210543, + 40.13390521217931, + 1.3034249027355285 + ] + }, + "periodicity_frob": { + "k_dominant_ac": 4.0, + "k_nsem": 3.0, + "k_nsem_power": 34.730175363045696, + "k_nsem_rank_in_ac": 3.0, + "k_nsem_ratio": 0.10972525438269988, + "total_ac_power": 316.5194335473012, + "power_spectrum": [ + 7.099748146989106e-30, + 154.77278852625855, + 34.65354634903779, + 34.730175363045696, + 47.567686082266206, + 43.0485030267296, + 22.98395067690211, + 21.26960126337535, + 17.252342294085743, + 14.654861154230446, + 21.09376000418449, + 15.629500356611684, + 9.059631136422345, + 8.155150551538757, + 6.937643047847938, + 4.502690197355615, + 2.9633898972142125, + 7.025906833237126, + 4.991095313216073 + ] + } +} \ No newline at end of file diff --git a/data/mi3_periodicity/gpt2-xl.json b/data/mi3_periodicity/gpt2-xl.json new file mode 100644 index 0000000000000000000000000000000000000000..fe8020ca231ff1ce440177df84ab027bc5828f93 --- /dev/null +++ b/data/mi3_periodicity/gpt2-xl.json @@ -0,0 +1,226 @@ +{ + "model": "gpt2-xl", + "N": 48, + "L_crit": 43, + "N_sem": 5, + "sr_curve": [ + 3.0052957418044377, + 4.190733397764279, + 3.9844778749153784, + 4.742144323138099, + 5.487735985789361, + 6.024953207324245, + 6.312635947467395, + 5.927214874958134, + 6.147061520141492, + 8.912540599478778, + 7.574532820508252, + 8.428949726032902, + 8.778703207288846, + 8.246854635259211, + 7.757720097985891, + 8.566503273916034, + 10.576764851082048, + 9.365914446895903, + 8.517875192047962, + 7.808563206248993, + 9.256986294407392, + 8.627361744835305, + 8.44072525056894, + 9.625278402884797, + 7.451156139598859, + 9.443563747594146, + 8.456589355789225, + 8.759770005423828, + 9.031517221674354, + 9.847088778453081, + 8.234535899593913, + 8.884081124344222, + 8.038374232438736, + 8.59403535919434, + 9.287073419463447, + 8.4247299245923, + 9.68373129290469, + 8.800401317758247, + 9.392212738040252, + 8.746496552641544, + 9.180510213501227, + 10.344749857443423, + 9.576668645082636, + 10.01476115963401, + 9.864034778742958, + 10.009398790526975, + 8.465146558744713, + 6.850672492399305 + ], + "frob_curve": [ + 0.41077608048915865, + 1.9284340691566468, + 3.133606698513031, + 3.560653681755066, + 4.071997008323669, + 3.9977815437316893, + 4.124623394012451, + 4.180846424102783, + 4.370722055435181, + 4.376299819946289, + 4.347878398895264, + 4.157600212097168, + 4.6784470272064205, + 4.586551275253296, + 4.583873338699341, + 4.46646637916565, + 5.0335115051269534, + 5.0603319263458255, + 4.770227365493774, + 4.717230911254883, + 4.805085496902466, + 4.42928144454956, + 4.091181364059448, + 3.974910135269165, + 3.612303147315979, + 3.6575075721740724, + 3.6522437381744384, + 3.4940106868743896, + 3.535118408203125, + 3.3711316204071045, + 3.3506378173828124, + 3.318546142578125, + 3.302897529602051, + 3.2507898807525635, + 3.496169033050537, + 3.1970692253112794, + 3.343003025054932, + 3.3962487983703613, + 3.1418694686889648, + 3.2417544746398925, + 3.367936706542969, + 3.595104990005493, + 3.7347336864471434, + 3.8008195781707763, + 4.3127646732330325, + 4.555081968307495, + 5.355396957397461, + 5.883860740661621 + ], + "spec_curve": [ + 0.24182699680328368, + 1.0088331055641175, + 1.63891273021698, + 1.697084596157074, + 1.8272642254829408, + 1.7690875434875488, + 1.7916849565505981, + 1.8929935598373413, + 1.9271667098999024, + 1.5420902919769288, + 1.6590872955322267, + 1.509448914527893, + 1.6759061193466187, + 1.7740382719039918, + 1.726573634147644, + 1.6041080713272096, + 1.6248615074157715, + 1.7891986989974975, + 1.8359429883956908, + 1.8478282928466796, + 1.76768390417099, + 1.6269321370124816, + 1.456128842830658, + 1.367765474319458, + 1.3652048897743225, + 1.2221035480499267, + 1.3716741514205932, + 1.257537717819214, + 1.2145938277244568, + 1.1094414949417115, + 1.221541187763214, + 1.2189280772209168, + 1.2721640133857728, + 1.2141858458518981, + 1.2644684052467345, + 1.1838785576820374, + 1.1286521339416504, + 1.3210419869422914, + 1.0470056414604187, + 1.1606328892707825, + 1.1938791680335998, + 1.1707167267799377, + 1.294056739807129, + 1.273641948699951, + 1.512703709602356, + 1.5043205046653747, + 1.9027538299560547, + 2.304016046524048 + ], + "periodicity_sr": { + "k_dominant_ac": 2.0, + "k_nsem": 5.0, + "k_nsem_power": 27.182511804459924, + "k_nsem_rank_in_ac": 13.0, + "k_nsem_ratio": 0.024977997638692313, + "total_ac_power": 1088.2582422200528, + "power_spectrum": [ + 2.6707674807162576e-26, + 715.0687660885052, + 357.855580965612, + 140.56885638501976, + 55.56968790762518, + 27.182511804459924, + 42.911084608788784, + 62.72325748769539, + 40.85365655711483, + 7.493694624005164, + 22.57786251205534, + 1.7465006088262922, + 65.10989462701167, + 31.02184474270982, + 11.287272020988171, + 20.113864678713906, + 1.683307546835651, + 27.375399993329392, + 31.497003685467494, + 16.696991455157452, + 3.4740980239949812, + 61.3301885468872, + 36.15480378397914, + 1.387032028836303, + 21.643847624939188 + ] + }, + "periodicity_frob": { + "k_dominant_ac": 2.0, + "k_nsem": 5.0, + "k_nsem_power": 52.202620820790955, + "k_nsem_rank_in_ac": 3.0, + "k_nsem_ratio": 0.09022596270323303, + "total_ac_power": 578.5764901450078, + "power_spectrum": [ + 1.7057144923141328e-27, + 280.26173833534295, + 67.67645876199053, + 9.4990292223341, + 66.13731086275538, + 52.202620820790955, + 41.212771571029194, + 36.08694686031306, + 37.69617844281605, + 32.47517721962243, + 39.50483034605495, + 27.75074106528551, + 10.28764603765761, + 19.72457458603383, + 22.889195420382176, + 13.46805307402927, + 18.961757724029955, + 9.354980736275186, + 20.83594529151453, + 15.155347961302242, + 10.364707794508256, + 5.727678389055702, + 12.414233366262787, + 7.185163847206159, + 1.9651407437580106 + ] + } +} \ No newline at end of file diff --git a/data/mi3b_circuit/gpt2_h0_L11.json b/data/mi3b_circuit/gpt2_h0_L11.json new file mode 100644 index 0000000000000000000000000000000000000000..ca8a5db1ea381bbd2df6ab7ffd420302fe12428c --- /dev/null +++ b/data/mi3b_circuit/gpt2_h0_L11.json @@ -0,0 +1,373 @@ +{ + "model": "gpt2", + "L_crit": 11, + "head": 0, + "ov_top_tokens": [ + " encount", + " challeng", + " nodd", + " mathemat", + " conclud", + " cryst", + " conduc", + " mosqu", + " destro", + " confir" + ], + "ov_top_vals": [ + 1.4117839336395264, + 1.4001333713531494, + 1.381157398223877, + 1.340135097503662, + 1.3175077438354492, + 1.31734037399292, + 1.2920207977294922, + 1.2823002338409424, + 1.2712907791137695, + 1.270348310470581 + ], + "pair_results": [ + { + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082, + "pos_first_A": 5, + "attn_first_A_short": 0.03729595988988876, + "attn_first_A_long": 0.0025198697112500668, + "attn_pattern_short": { + "attn_last": 0.12043865025043488, + "attn_prev": 0.16115061938762665, + "attn_first": 0.00917611550539732, + "top_pos": [ + 15, + 1, + 16 + ], + "top_attn": [ + 0.16115061938762665, + 0.13423262536525726, + 0.12043865025043488 + ] + }, + "attn_pattern_long": { + "attn_last": 0.024424100294709206, + "attn_prev": 0.031246626749634743, + "attn_first": 0.004257743246853352, + "top_pos": [ + 175, + 157, + 205 + ], + "top_attn": [ + 0.04705679789185524, + 0.040128640830516815, + 0.031246626749634743 + ] + } + }, + { + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184, + "pos_first_A": 5, + "attn_first_A_short": 0.056623075157403946, + "attn_first_A_long": 0.005963335279375315, + "attn_pattern_short": { + "attn_last": 0.013525108806788921, + "attn_prev": 0.049177445471286774, + "attn_first": 0.06575673818588257, + "top_pos": [ + 6, + 11, + 12 + ], + "top_attn": [ + 0.2076505720615387, + 0.12938766181468964, + 0.09972482919692993 + ] + }, + "attn_pattern_long": { + "attn_last": 0.007851049304008484, + "attn_prev": 0.03519149869680405, + "attn_first": 0.013331804424524307, + "top_pos": [ + 203, + 188, + 205 + ], + "top_attn": [ + 0.03899011388421059, + 0.03769507259130478, + 0.03519149869680405 + ] + } + }, + { + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875, + "pos_first_A": 5, + "attn_first_A_short": 0.021057741716504097, + "attn_first_A_long": 0.004016018006950617, + "attn_pattern_short": { + "attn_last": 0.03277069702744484, + "attn_prev": 0.03916487470269203, + "attn_first": 0.3385145962238312, + "top_pos": [ + 0, + 2, + 7 + ], + "top_attn": [ + 0.3385145962238312, + 0.0758148729801178, + 0.05873050540685654 + ] + }, + "attn_pattern_long": { + "attn_last": 0.012658486142754555, + "attn_prev": 0.01410688366740942, + "attn_first": 0.21020843088626862, + "top_pos": [ + 0, + 171, + 179 + ], + "top_attn": [ + 0.21020843088626862, + 0.016973674297332764, + 0.01670595444738865 + ] + } + }, + { + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074, + "pos_first_A": 5, + "attn_first_A_short": 0.07590733468532562, + "attn_first_A_long": 0.00352013623341918, + "attn_pattern_short": { + "attn_last": 0.05383799970149994, + "attn_prev": 0.04145648330450058, + "attn_first": 0.027250060811638832, + "top_pos": [ + 6, + 1, + 8 + ], + "top_attn": [ + 0.17372184991836548, + 0.12140417098999023, + 0.09570734202861786 + ] + }, + "attn_pattern_long": { + "attn_last": 0.007243819534778595, + "attn_prev": 0.014199996367096901, + "attn_first": 0.033251747488975525, + "top_pos": [ + 0, + 200, + 195 + ], + "top_attn": [ + 0.033251747488975525, + 0.02584499679505825, + 0.024387236684560776 + ] + } + }, + { + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129, + "pos_first_A": 5, + "attn_first_A_short": 0.14539474248886108, + "attn_first_A_long": 0.010047012940049171, + "attn_pattern_short": { + "attn_last": 0.17167329788208008, + "attn_prev": 0.03896866738796234, + "attn_first": 0.028675192967057228, + "top_pos": [ + 16, + 2, + 5 + ], + "top_attn": [ + 0.17167329788208008, + 0.17065678536891937, + 0.14539474248886108 + ] + }, + "attn_pattern_long": { + "attn_last": 0.058717213571071625, + "attn_prev": 0.01719764433801174, + "attn_first": 0.0086270896717906, + "top_pos": [ + 206, + 192, + 190 + ], + "top_attn": [ + 0.058717213571071625, + 0.02709386497735977, + 0.027025654911994934 + ] + } + }, + { + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609, + "pos_first_A": 5, + "attn_first_A_short": 0.27401113510131836, + "attn_first_A_long": 0.018537333235144615, + "attn_pattern_short": { + "attn_last": 0.17748388648033142, + "attn_prev": 0.024510622024536133, + "attn_first": 0.009095129556953907, + "top_pos": [ + 5, + 16, + 11 + ], + "top_attn": [ + 0.27401113510131836, + 0.17748388648033142, + 0.09921993315219879 + ] + }, + "attn_pattern_long": { + "attn_last": 0.0312521830201149, + "attn_prev": 0.0005667301593348384, + "attn_first": 0.0037122047506272793, + "top_pos": [ + 197, + 206, + 175 + ], + "top_attn": [ + 0.04358495771884918, + 0.0312521830201149, + 0.029594330117106438 + ] + } + }, + { + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078, + "pos_first_A": 5, + "attn_first_A_short": 0.056949034333229065, + "attn_first_A_long": 0.006423028651624918, + "attn_pattern_short": { + "attn_last": 0.07360777258872986, + "attn_prev": 0.08337710052728653, + "attn_first": 0.0740671306848526, + "top_pos": [ + 2, + 1, + 8 + ], + "top_attn": [ + 0.10304904729127884, + 0.10088726133108139, + 0.0898955836892128 + ] + }, + "attn_pattern_long": { + "attn_last": 0.015702001750469208, + "attn_prev": 0.013866969384253025, + "attn_first": 0.16084440052509308, + "top_pos": [ + 0, + 196, + 206 + ], + "top_attn": [ + 0.16084440052509308, + 0.01683749258518219, + 0.015702001750469208 + ] + } + }, + { + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984, + "pos_first_A": 5, + "attn_first_A_short": 0.029223106801509857, + "attn_first_A_long": 0.0028024364728480577, + "attn_pattern_short": { + "attn_last": 0.102736696600914, + "attn_prev": 0.0680849626660347, + "attn_first": 0.039659906178712845, + "top_pos": [ + 11, + 10, + 16 + ], + "top_attn": [ + 0.16312819719314575, + 0.13366363942623138, + 0.102736696600914 + ] + }, + "attn_pattern_long": { + "attn_last": 0.005859490483999252, + "attn_prev": 0.011134556494653225, + "attn_first": 0.09949424117803574, + "top_pos": [ + 0, + 203, + 185 + ], + "top_attn": [ + 0.09949424117803574, + 0.01816416159272194, + 0.017462285235524178 + ] + } + }, + { + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961, + "pos_first_A": 5, + "attn_first_A_short": 0.01489474531263113, + "attn_first_A_long": 0.002471849787980318, + "attn_pattern_short": { + "attn_last": 0.09140582382678986, + "attn_prev": 0.05755604803562164, + "attn_first": 0.06188209354877472, + "top_pos": [ + 1, + 14, + 13 + ], + "top_attn": [ + 0.1304914951324463, + 0.11744718998670578, + 0.10083449631929398 + ] + }, + "attn_pattern_long": { + "attn_last": 0.033711254596710205, + "attn_prev": 0.023164527490735054, + "attn_first": 0.018670279532670975, + "top_pos": [ + 206, + 200, + 202 + ], + "top_attn": [ + 0.033711254596710205, + 0.03164033964276314, + 0.024893512949347496 + ] + } + } + ] +} \ No newline at end of file diff --git a/data/mi3c_circuit/gpt2medium_targets.json b/data/mi3c_circuit/gpt2medium_targets.json new file mode 100644 index 0000000000000000000000000000000000000000..0a4498f70c6aba86d434a30fe7eb808fd18e4ecd --- /dev/null +++ b/data/mi3c_circuit/gpt2medium_targets.json @@ -0,0 +1,82 @@ +[ + { + "layer": 1, + "head": 8, + "label": "top-1 MI-2b", + "mean_attn_short": 0.022860591299831867, + "mean_attn_long": 0.003218296985141933, + "ratio": 7.103297294314412, + "induction": "YES", + "ov_top": [ + " Thumbnails", + " Each", + " guiActiveUnfocused", + " https", + " The", + " Houses", + " Though", + " EVERY" + ], + "n_pairs": 10 + }, + { + "layer": 7, + "head": 0, + "label": "top-2 MI-2b", + "mean_attn_short": 0.00786478203954175, + "mean_attn_long": 0.001453155224226066, + "ratio": 5.412173308599796, + "induction": "YES", + "ov_top": [ + "surv", + " perpet", + "Program", + " Juda", + " athlet", + "Community", + "atever", + "paste" + ], + "n_pairs": 10 + }, + { + "layer": 22, + "head": 9, + "label": "near L_crit-1", + "mean_attn_short": 0.016976462613092734, + "mean_attn_long": 0.002230018798582023, + "ratio": 7.612665192434878, + "induction": "YES", + "ov_top": [ + " Flowers", + " Collect", + " Doct", + "arted", + " Powell", + "Barn", + "learn", + "anian" + ], + "n_pairs": 10 + }, + { + "layer": 23, + "head": 5, + "label": "L_crit", + "mean_attn_short": 0.10905305915512145, + "mean_attn_long": 0.017094450870354196, + "ratio": 6.379438344513399, + "induction": "YES", + "ov_top": [ + "The", + "This", + "5", + "1", + "esp", + " followed", + "When", + "For" + ], + "n_pairs": 10 + } +] \ No newline at end of file diff --git a/data/mi4_formula/gpt2_formula_validation.json b/data/mi4_formula/gpt2_formula_validation.json new file mode 100644 index 0000000000000000000000000000000000000000..cf4185c83352c6ad9ba712ee3bd079b2c3706e54 --- /dev/null +++ b/data/mi4_formula/gpt2_formula_validation.json @@ -0,0 +1,6158 @@ +[ + { + "delta_attn": 0.0019324274035170674, + "spec_norm": 6.036941609910174, + "frob": 31.60514503796959, + "SR": 27.408270051027085, + "cos_in": -0.49467897415161133, + "cos_out": -0.04785465449094772, + "align_raw": -0.13553966581821442, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": -0.041183945930712014, + "R_pred_norm": 0.06392851510487148, + "R_pred_raw": 0.02999299721742229, + "layer": 9, + "head": 0, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0009424422751180828, + "spec_norm": 10.93353757745038, + "frob": 48.79774054627811, + "SR": 19.919481210910376, + "cos_in": 0.5898611545562744, + "cos_out": 0.07278033345937729, + "align_raw": 0.20613409578800201, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": -0.04743247144106997, + "R_pred_norm": 0.10240145234636151, + "R_pred_raw": 0.026526576618189336, + "layer": 9, + "head": 1, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0006542953196913004, + "spec_norm": 5.876704192031835, + "frob": 34.24818760886753, + "SR": 33.963115826977834, + "cos_in": 0.21765553951263428, + "cos_out": 0.008959044702351093, + "align_raw": 0.02537531405687332, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": 0.022293073835844054, + "R_pred_norm": 0.0017356649383063341, + "R_pred_raw": 0.0008365305041312028, + "layer": 9, + "head": 2, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0024833985735313036, + "spec_norm": 7.177969011896205, + "frob": 22.909361998700444, + "SR": 10.186449376848554, + "cos_in": 0.6527060270309448, + "cos_out": 0.02239791676402092, + "align_raw": 0.06343899667263031, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": 0.04768551427689113, + "R_pred_norm": 0.060325380991675025, + "R_pred_raw": 0.023803848288910807, + "layer": 9, + "head": 3, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": -0.0009680329822003841, + "spec_norm": 10.769035341774249, + "frob": 47.84954141139965, + "SR": 19.74249129875483, + "cos_in": -0.55316561460495, + "cos_out": -0.033774927258491516, + "align_raw": -0.09566183388233185, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": 0.048655239725134795, + "R_pred_norm": -0.04508617297196432, + "R_pred_raw": -0.011857983740935225, + "layer": 9, + "head": 4, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0017811749130487442, + "spec_norm": 13.503818678302977, + "frob": 53.306145668352585, + "SR": 15.582651852687384, + "cos_in": 0.6546218991279602, + "cos_out": -0.0035468144342303276, + "align_raw": -0.010045873001217842, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": 0.01635554613338242, + "R_pred_norm": -0.012927632036537223, + "R_pred_raw": -0.0027115124360703713, + "layer": 9, + "head": 5, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0003556644660420716, + "spec_norm": 8.568407234665175, + "frob": 51.29468564471981, + "SR": 35.83806450218393, + "cos_in": 0.26377540826797485, + "cos_out": 0.02167876996099949, + "align_raw": 0.0614020898938179, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": -0.1033402453477241, + "R_pred_norm": 0.004034004100854474, + "R_pred_raw": 0.0013334746141519132, + "layer": 9, + "head": 6, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0027305237017571926, + "spec_norm": 9.229841381292971, + "frob": 40.66509439732988, + "SR": 19.411321133223833, + "cos_in": 0.04938467592000961, + "cos_out": 0.019487671554088593, + "align_raw": 0.055195197463035583, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": 0.011289791813685295, + "R_pred_norm": 0.005614604448678594, + "R_pred_raw": 0.0017229245839301256, + "layer": 9, + "head": 7, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0025592055171728134, + "spec_norm": 9.276381529047894, + "frob": 39.05353959730315, + "SR": 17.724075819383255, + "cos_in": -0.5779554843902588, + "cos_out": -0.03539680317044258, + "align_raw": -0.10025643557310104, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": 0.033946920826226884, + "R_pred_norm": 0.11242658360174224, + "R_pred_raw": 0.034327219754903555, + "layer": 9, + "head": 8, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": -0.0010679622646421194, + "spec_norm": 8.40992628067492, + "frob": 51.80072336759642, + "SR": 37.93912156776617, + "cos_in": -0.04591713100671768, + "cos_out": -0.001957796048372984, + "align_raw": -0.0055451709777116776, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": 0.009148722915753365, + "R_pred_norm": -0.0001869033589582962, + "R_pred_raw": -6.294661948389545e-05, + "layer": 9, + "head": 9, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.000363178810403042, + "spec_norm": 6.252881415873531, + "frob": 31.1179750167106, + "SR": 24.766365082827907, + "cos_in": 0.522895872592926, + "cos_out": 0.061899583786726, + "align_raw": 0.17532037198543549, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": 0.12410118562603165, + "R_pred_norm": 0.017014941276219827, + "R_pred_raw": 0.00770716961291695, + "layer": 9, + "head": 10, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.005978486442472786, + "spec_norm": 8.934042094947666, + "frob": 40.733481961825646, + "SR": 20.787730739716768, + "cos_in": 0.5184394717216492, + "cos_out": 0.037499088793992996, + "align_raw": 0.10620839893817902, + "norm_xA": 135.22735595703125, + "scale": 231.48711248355292, + "R_obs": -0.23382872191020918, + "R_pred_norm": 0.2403725302238346, + "R_pred_raw": 0.07620351211369868, + "layer": 9, + "head": 11, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": -9.574694558978081e-05, + "spec_norm": 9.550612297201935, + "frob": 59.05110670533618, + "SR": 38.22906285048873, + "cos_in": -0.08788739144802094, + "cos_out": -0.0017648841021582484, + "align_raw": -0.004998695105314255, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": -0.01250766249447616, + "R_pred_norm": -4.012793659946382e-05, + "R_pred_raw": -1.1900249585040897e-05, + "layer": 10, + "head": 0, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.00014232592366170138, + "spec_norm": 8.474343117432328, + "frob": 50.37616313793658, + "SR": 35.337684185342425, + "cos_in": 0.3566926121711731, + "cos_out": 0.04222666099667549, + "align_raw": 0.1196005716919899, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": -0.06255300528220267, + "R_pred_norm": 0.005139475856365109, + "R_pred_raw": 0.0017177474005048011, + "layer": 10, + "head": 1, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.012165636289864779, + "spec_norm": 9.941501795341454, + "frob": 55.272725812556054, + "SR": 30.911335920222314, + "cos_in": -0.5141991376876831, + "cos_out": -0.04636513069272041, + "align_raw": -0.1313207447528839, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": -0.04157493792541632, + "R_pred_norm": 0.8157509253963774, + "R_pred_raw": 0.2324060495605666, + "layer": 10, + "head": 2, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.01791934808716178, + "spec_norm": 9.503399692367497, + "frob": 41.48036998087167, + "SR": 19.051415661602288, + "cos_in": -0.2338908612728119, + "cos_out": -0.018761340528726578, + "align_raw": -0.05313822627067566, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": -0.05151952137318784, + "R_pred_norm": 0.211410347170398, + "R_pred_raw": 0.06300723649539056, + "layer": 10, + "head": 3, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.00034046424480038695, + "spec_norm": 19.162915910461177, + "frob": 41.36431396417161, + "SR": 4.6593835707170745, + "cos_in": -0.7622868418693542, + "cos_out": 0.10776429623365402, + "align_raw": 0.305216521024704, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": 0.08565010232218814, + "R_pred_norm": -0.15162623207096843, + "R_pred_raw": -0.022410208116726803, + "layer": 10, + "head": 4, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.004566239382256754, + "spec_norm": 10.470172842324263, + "frob": 56.42047950711249, + "SR": 29.03794266776303, + "cos_in": 0.5138160586357117, + "cos_out": 0.0028477299492806196, + "align_raw": 0.008065704256296158, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": 0.2783201825852674, + "R_pred_norm": 0.019790962765836185, + "R_pred_raw": 0.005353731191968363, + "layer": 10, + "head": 5, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0016302854055538774, + "spec_norm": 9.375325108295202, + "frob": 56.18359795961362, + "SR": 35.91256474485841, + "cos_in": -0.094211645424366, + "cos_out": 0.04121171310544014, + "align_raw": 0.1167263612151146, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": -0.02035117413771974, + "R_pred_norm": -0.016788935215736656, + "R_pred_raw": -0.005072068333467722, + "layer": 10, + "head": 6, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": -0.0030555478297173977, + "spec_norm": 11.000900231717544, + "frob": 46.6323301878643, + "SR": 17.968746540541265, + "cos_in": -0.632621169090271, + "cos_out": 0.0064896754920482635, + "align_raw": 0.018380511552095413, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": 0.11643725276949986, + "R_pred_norm": 0.039041988830072004, + "R_pred_raw": 0.010051672265496862, + "layer": 10, + "head": 7, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0036166319623589516, + "spec_norm": 15.539008982908092, + "frob": 50.32241507503763, + "SR": 10.487604850112564, + "cos_in": -0.7421846389770508, + "cos_out": -0.0774403065443039, + "align_raw": -0.2193371206521988, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": -0.05255618073284227, + "R_pred_norm": 0.9138072048856594, + "R_pred_raw": 0.16656215635889995, + "layer": 10, + "head": 8, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0015984506644599605, + "spec_norm": 9.145337160130344, + "frob": 32.93676432705462, + "SR": 12.970670360230072, + "cos_in": -0.06463571637868881, + "cos_out": 0.0029971201438456774, + "align_raw": 0.008488859981298447, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": 0.0025434886336088047, + "R_pred_norm": -0.0008011680027736366, + "R_pred_raw": -0.0002481241820403265, + "layer": 10, + "head": 9, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.008568509947508574, + "spec_norm": 39.854820765890416, + "frob": 74.28405022067321, + "SR": 3.4739969204351477, + "cos_in": -0.7370656728744507, + "cos_out": -0.0676150992512703, + "align_raw": -0.19151118397712708, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": -0.008088391806941478, + "R_pred_norm": 4.814863284049073, + "R_pred_raw": 0.34217915619563183, + "layer": 10, + "head": 10, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0022515251766890287, + "spec_norm": 7.553501039928774, + "frob": 30.829615793274776, + "SR": 16.658643653607324, + "cos_in": -0.6901311278343201, + "cos_out": 0.004927599336951971, + "align_raw": 0.013956308364868164, + "norm_xA": 165.26681518554688, + "scale": 282.9097527338442, + "R_obs": -0.02329789877421775, + "R_pred_norm": -0.01636213699596551, + "R_pred_raw": -0.006135174218483339, + "layer": 10, + "head": 11, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0347760901786387, + "spec_norm": 61.65889700666178, + "frob": 86.9711978635025, + "SR": 1.9895708090537703, + "cos_in": -0.4610164165496826, + "cos_out": -0.11062665283679962, + "align_raw": -0.3133275508880615, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": 0.20226938268003677, + "R_pred_norm": 43.086182699467386, + "R_pred_raw": 1.979160791938781, + "layer": 11, + "head": 0, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.007908696308732033, + "spec_norm": 18.265131282089193, + "frob": 77.23712031080814, + "SR": 17.881607190408033, + "cos_in": 0.4119522273540497, + "cos_out": 0.02932600863277912, + "align_raw": 0.08306033909320831, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": 0.03229316345405377, + "R_pred_norm": 0.6875635754572671, + "R_pred_raw": 0.10661806712335799, + "layer": 11, + "head": 1, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.011817353079095483, + "spec_norm": 18.919704826262237, + "frob": 106.23267321956556, + "SR": 31.527352839346545, + "cos_in": 0.1676473766565323, + "cos_out": -0.022341333329677582, + "align_raw": -0.0632781833410263, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": 0.03712138401496393, + "R_pred_norm": -0.32993300990364205, + "R_pred_raw": -0.04939196754246335, + "layer": 11, + "head": 2, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.03202590555883944, + "spec_norm": 13.58022967182268, + "frob": 68.92652762860973, + "SR": 25.760754015986006, + "cos_in": -0.1688285917043686, + "cos_out": -0.017369527369737625, + "align_raw": -0.04919594153761864, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": 0.013419432970644657, + "R_pred_norm": 0.5024905322314902, + "R_pred_raw": 0.10480018368915531, + "layer": 11, + "head": 3, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.007182596251368523, + "spec_norm": 48.044989800413894, + "frob": 81.35082986381424, + "SR": 2.8670004694928535, + "cos_in": -0.6708555817604065, + "cos_out": 0.05836089327931404, + "align_raw": 0.1652991622686386, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": 0.038034787025557085, + "R_pred_norm": -5.323115371190978, + "R_pred_raw": -0.313809795188556, + "layer": 11, + "head": 4, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.012918745400384068, + "spec_norm": 19.05751647844419, + "frob": 80.7581869882243, + "SR": 17.95727829022412, + "cos_in": -0.3776354193687439, + "cos_out": -0.07049424946308136, + "align_raw": -0.19965939223766327, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": -0.11820120621533703, + "R_pred_norm": 2.5822507139664337, + "R_pred_raw": 0.38376746725884714, + "layer": 11, + "head": 5, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.013174987398087978, + "spec_norm": 18.683054315466933, + "frob": 97.40623997433379, + "SR": 27.181774531023397, + "cos_in": -0.5670954585075378, + "cos_out": 0.03720882907509804, + "align_raw": 0.10538871586322784, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": -0.036551629500760165, + "R_pred_norm": -2.046376822625472, + "R_pred_raw": -0.3102314632624666, + "layer": 11, + "head": 6, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.03462807089090347, + "spec_norm": 16.053417680926426, + "frob": 76.0262341645961, + "SR": 22.42807228285055, + "cos_in": 0.03611220791935921, + "cos_out": -0.025468189269304276, + "align_raw": -0.07213358581066132, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": 0.0026202177515674786, + "R_pred_norm": -0.2014342263939704, + "R_pred_raw": -0.035539000758276124, + "layer": 11, + "head": 7, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.02660660771653056, + "spec_norm": 334.85910049974757, + "frob": 338.5618019653133, + "SR": 1.022237249959398, + "cos_in": 0.45048779249191284, + "cos_out": 0.023359255865216255, + "align_raw": 0.0661589503288269, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": -0.05386547334141368, + "R_pred_norm": 36.938477749951325, + "R_pred_raw": 0.31242551192835527, + "layer": 11, + "head": 8, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0028919518226757646, + "spec_norm": 18.626947288356273, + "frob": 90.40253945893888, + "SR": 23.55471692205203, + "cos_in": -0.33218351006507874, + "cos_out": -0.09464764595031738, + "align_raw": -0.26806503534317017, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": -0.026960489755925934, + "R_pred_norm": 0.6672767249131459, + "R_pred_raw": 0.10145995033429103, + "layer": 11, + "head": 9, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.005626028403639793, + "spec_norm": 14.325500713871225, + "frob": 62.8168530829904, + "SR": 19.22793877092154, + "cos_in": -0.061644650995731354, + "cos_out": 0.002191105391830206, + "align_raw": 0.006205707788467407, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": 0.04696719913004397, + "R_pred_norm": -0.004288997049430909, + "R_pred_raw": -0.0008479573152343409, + "layer": 11, + "head": 10, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.0052275962952990085, + "spec_norm": 62.309257767330415, + "frob": 95.02363501222999, + "SR": 2.3257237279019654, + "cos_in": 0.3020072877407074, + "cos_out": 0.06152227148413658, + "align_raw": 0.1742546111345291, + "norm_xA": 230.1564178466797, + "scale": 393.99013764504286, + "R_obs": 0.04857851060717612, + "R_pred_norm": 2.384458427604526, + "R_pred_raw": 0.10838996960878391, + "layer": 11, + "head": 11, + "word_A": "Paris", + "word_B": " is", + "gap": 0.5841679573059082 + }, + { + "delta_attn": 0.01731350872432813, + "spec_norm": 6.036941609910174, + "frob": 31.60514503796959, + "SR": 27.408270051027085, + "cos_in": -0.5425602793693542, + "cos_out": -0.04785465449094772, + "align_raw": -0.13553966581821442, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": 0.013613466089057726, + "R_pred_norm": 0.3054740079611469, + "R_pred_raw": 0.14331759553219067, + "layer": 9, + "head": 0, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.043387098237872124, + "spec_norm": 10.93353757745038, + "frob": 48.79774054627811, + "SR": 19.919481210910376, + "cos_in": 0.6341872811317444, + "cos_out": 0.07278033345937729, + "align_raw": 0.20613409578800201, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": 0.0046219812232773504, + "R_pred_norm": 2.4646365948134346, + "R_pred_raw": 0.638451603666487, + "layer": 9, + "head": 1, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.00835888771689497, + "spec_norm": 5.876704192031835, + "frob": 34.24818760886753, + "SR": 33.963115826977834, + "cos_in": 0.27148211002349854, + "cos_out": 0.008959044702351093, + "align_raw": 0.02537531405687332, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": -0.0039935230965202064, + "R_pred_norm": 0.01344885336347618, + "R_pred_raw": 0.0064818824393108355, + "layer": 9, + "head": 2, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.0016171047927855398, + "spec_norm": 7.177969011896205, + "frob": 22.909361998700444, + "SR": 10.186449376848554, + "cos_in": 0.6952016949653625, + "cos_out": 0.02239791676402092, + "align_raw": 0.06343899667263031, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": 0.03977108145448192, + "R_pred_norm": 0.020345024100416872, + "R_pred_raw": 0.008027962014651677, + "layer": 9, + "head": 3, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.0048269172257278115, + "spec_norm": 10.769035341774249, + "frob": 47.84954141139965, + "SR": 19.74249129875483, + "cos_in": -0.5588142275810242, + "cos_out": -0.033774927258491516, + "align_raw": -0.09566183388233185, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": 0.015892799295057516, + "R_pred_norm": 0.11043546074572229, + "R_pred_raw": 0.02904531060464525, + "layer": 9, + "head": 4, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.01645368098979816, + "spec_norm": 13.503818678302977, + "frob": 53.306145668352585, + "SR": 15.582651852687384, + "cos_in": 0.6846268177032471, + "cos_out": -0.0035468144342303276, + "align_raw": -0.010045873001217842, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": -0.010616565021909854, + "R_pred_norm": -0.06073122782001534, + "R_pred_raw": -0.012738100761715653, + "layer": 9, + "head": 5, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.1439045574516058, + "spec_norm": 8.568407234665175, + "frob": 51.29468564471981, + "SR": 35.83806450218393, + "cos_in": 0.2534092664718628, + "cos_out": 0.02167876996099949, + "align_raw": 0.0614020898938179, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": -0.0447205800348628, + "R_pred_norm": 0.7624858574803817, + "R_pred_raw": 0.25204623227442324, + "layer": 9, + "head": 6, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.0078060644591460004, + "spec_norm": 9.229841381292971, + "frob": 40.66509439732988, + "SR": 19.411321133223833, + "cos_in": 0.03249192610383034, + "cos_out": 0.019487671554088593, + "align_raw": 0.055195197463035583, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": 0.02060623524573609, + "R_pred_norm": 0.0051352524160417955, + "R_pred_raw": 0.0015758283086829549, + "layer": 9, + "head": 7, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.005989057128317654, + "spec_norm": 9.276381529047894, + "frob": 39.05353959730315, + "SR": 17.724075819383255, + "cos_in": -0.6126797795295715, + "cos_out": -0.03539680317044258, + "align_raw": -0.10025643557310104, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": 0.017816475227830312, + "R_pred_norm": 0.13562341106192272, + "R_pred_raw": 0.04140990934958992, + "layer": 9, + "head": 8, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.13575321075040847, + "spec_norm": 8.40992628067492, + "frob": 51.80072336759642, + "SR": 37.93912156776617, + "cos_in": -0.0468657985329628, + "cos_out": -0.001957796048372984, + "align_raw": -0.0055451709777116776, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": -0.09079656602967198, + "R_pred_norm": 0.011791411190910837, + "R_pred_raw": 0.003971193870186275, + "layer": 9, + "head": 9, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.0013487305786838988, + "spec_norm": 6.252881415873531, + "frob": 31.1179750167106, + "SR": 24.766365082827907, + "cos_in": 0.5318865776062012, + "cos_out": 0.061899583786726, + "align_raw": 0.17532037198543549, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": 0.007111582135866283, + "R_pred_norm": 0.03125447245181303, + "R_pred_raw": 0.01415717612173152, + "layer": 9, + "head": 10, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.0012331041798461229, + "spec_norm": 8.934042094947666, + "frob": 40.733481961825646, + "SR": 20.787730739716768, + "cos_in": 0.5427817106246948, + "cos_out": 0.037499088793992996, + "align_raw": 0.10620839893817902, + "norm_xA": 137.3345489501953, + "scale": 112.56412936050833, + "R_obs": -0.07868389546719848, + "R_pred_norm": 0.025240252840075156, + "R_pred_raw": 0.008001729279383219, + "layer": 9, + "head": 11, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.07847416010918096, + "spec_norm": 9.550612297201935, + "frob": 59.05110670533618, + "SR": 38.22906285048873, + "cos_in": -0.0834273248910904, + "cos_out": -0.0017648841021582484, + "align_raw": -0.004998695105314255, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": 0.047268805772708936, + "R_pred_norm": 0.014884977634030044, + "R_pred_raw": 0.004414255103141184, + "layer": 10, + "head": 0, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.16938027390278876, + "spec_norm": 8.474343117432328, + "frob": 50.37616313793658, + "SR": 35.337684185342425, + "cos_in": 0.35882529616355896, + "cos_out": 0.04222666099667549, + "align_raw": 0.1196005716919899, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": 0.03219988068859921, + "R_pred_norm": 2.9336240289127367, + "R_pred_raw": 0.9804939629169233, + "layer": 10, + "head": 1, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.012466326064895838, + "spec_norm": 9.941501795341454, + "frob": 55.272725812556054, + "SR": 30.911335920222314, + "cos_in": -0.5072546005249023, + "cos_out": -0.04636513069272041, + "align_raw": -0.1313207447528839, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": -0.025710503364795974, + "R_pred_norm": 0.3931640007163032, + "R_pred_raw": 0.11201175431275777, + "layer": 10, + "head": 2, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.20763982902280986, + "spec_norm": 9.503399692367497, + "frob": 41.48036998087167, + "SR": 19.051415661602288, + "cos_in": -0.21336863934993744, + "cos_out": -0.018761340528726578, + "align_raw": -0.05313822627067566, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": 0.009177520977929877, + "R_pred_norm": 1.0654912737820834, + "R_pred_raw": 0.3175514423466347, + "layer": 10, + "head": 3, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.01089627225883305, + "spec_norm": 19.162915910461177, + "frob": 41.36431396417161, + "SR": 4.6593835707170745, + "cos_in": -0.774443507194519, + "cos_out": 0.10776429623365402, + "align_raw": 0.305216521024704, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": 0.024328520817399668, + "R_pred_norm": -2.3505525544625736, + "R_pred_raw": -0.34740935796752515, + "layer": 10, + "head": 4, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.0029065730959700886, + "spec_norm": 10.470172842324263, + "frob": 56.42047950711249, + "SR": 29.03794266776303, + "cos_in": 0.5266958475112915, + "cos_out": 0.0028477299492806196, + "align_raw": 0.008065704256296158, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": 0.009773930866879754, + "R_pred_norm": 0.006156864319805611, + "R_pred_raw": 0.0016655175871767498, + "layer": 10, + "head": 5, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.06250105221988633, + "spec_norm": 9.375325108295202, + "frob": 56.18359795961362, + "SR": 35.91256474485841, + "cos_in": -0.05424985662102699, + "cos_out": 0.04121171310544014, + "align_raw": 0.1167263612151146, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": -0.0171434622338777, + "R_pred_norm": -0.176709018097041, + "R_pred_raw": -0.05338517323529084, + "layer": 10, + "head": 6, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.010916162107605487, + "spec_norm": 11.000900231717544, + "frob": 46.6323301878643, + "SR": 17.968746540541265, + "cos_in": -0.6419457197189331, + "cos_out": 0.0064896754920482635, + "align_raw": 0.018380511552095413, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": -0.004787694062671025, + "R_pred_norm": -0.06748158528350987, + "R_pred_raw": -0.017373673820211644, + "layer": 10, + "head": 7, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.02620177436619997, + "spec_norm": 15.539008982908092, + "frob": 50.32241507503763, + "SR": 10.487604850112564, + "cos_in": -0.7541653513908386, + "cos_out": -0.0774403065443039, + "align_raw": -0.2193371206521988, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": 0.06330308631272048, + "R_pred_norm": 3.2074023639874105, + "R_pred_raw": 0.5846220638227759, + "layer": 10, + "head": 8, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.0006616243916823805, + "spec_norm": 9.145337160130344, + "frob": 32.93676432705462, + "SR": 12.970670360230072, + "cos_in": -0.09141254425048828, + "cos_out": 0.0029971201438456774, + "align_raw": 0.008488859981298447, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": 0.03616526386899597, + "R_pred_norm": -0.0002236078077204663, + "R_pred_raw": -6.925202229294151e-05, + "layer": 10, + "head": 9, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.174408063525334, + "spec_norm": 39.854820765890416, + "frob": 74.28405022067321, + "SR": 3.4739969204351477, + "cos_in": -0.7481874823570251, + "cos_out": -0.0676150992512703, + "align_raw": -0.19151118397712708, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": 0.013438373277622899, + "R_pred_norm": 47.431551831787495, + "R_pred_raw": 3.3708305771875775, + "layer": 10, + "head": 10, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.10733760334551334, + "spec_norm": 7.553501039928774, + "frob": 30.829615793274776, + "SR": 16.658643653607324, + "cos_in": -0.7259609699249268, + "cos_out": 0.004927599336951971, + "align_raw": 0.013956308364868164, + "norm_xA": 164.5680694580078, + "scale": 134.8856613334657, + "R_obs": 0.012203343625537965, + "R_pred_norm": -0.39121422090184427, + "R_pred_raw": -0.14669033773356516, + "layer": 10, + "head": 11, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.05065973987802863, + "spec_norm": 61.65889700666178, + "frob": 86.9711978635025, + "SR": 1.9895708090537703, + "cos_in": -0.4887551963329315, + "cos_out": -0.11062665283679962, + "align_raw": -0.3133275508880615, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": 0.1640580559927482, + "R_pred_norm": 36.58172098442966, + "R_pred_raw": 1.6803787975146454, + "layer": 11, + "head": 0, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.03269101120531559, + "spec_norm": 18.265131282089193, + "frob": 77.23712031080814, + "SR": 17.881607190408033, + "cos_in": 0.44302278757095337, + "cos_out": 0.02932600863277912, + "align_raw": 0.08306033909320831, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": -0.022161747275893696, + "R_pred_norm": 1.6802907703278511, + "R_pred_raw": 0.2605567841758192, + "layer": 11, + "head": 1, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.08425065746996552, + "spec_norm": 18.919704826262237, + "frob": 106.23267321956556, + "SR": 31.527352839346545, + "cos_in": 0.1972723752260208, + "cos_out": -0.022341333329677582, + "align_raw": -0.0632781833410263, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": -0.0001680578324039625, + "R_pred_norm": -1.5216588033008107, + "R_pred_raw": -0.2277969162445653, + "layer": 11, + "head": 2, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.018517754855565727, + "spec_norm": 13.58022967182268, + "frob": 68.92652762860973, + "SR": 25.760754015986006, + "cos_in": -0.20270375907421112, + "cos_out": -0.017369527369737625, + "align_raw": -0.04919594153761864, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": 0.0005119901405795137, + "R_pred_norm": 0.19177832231605527, + "R_pred_raw": 0.039997576306694614, + "layer": 11, + "head": 3, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.03509254474192858, + "spec_norm": 48.044989800413894, + "frob": 81.35082986381424, + "SR": 2.8670004694928535, + "cos_in": -0.7121192812919617, + "cos_out": 0.05836089327931404, + "align_raw": 0.1652991622686386, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": 0.030374694462258506, + "R_pred_norm": -15.177208805286929, + "R_pred_raw": -0.8947310840748198, + "layer": 11, + "head": 4, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.08547548647038639, + "spec_norm": 19.05751647844419, + "frob": 80.7581869882243, + "SR": 17.95727829022412, + "cos_in": -0.3976205587387085, + "cos_out": -0.07049424946308136, + "align_raw": -0.19965939223766327, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": -0.032739229080965424, + "R_pred_norm": 9.889735424282232, + "R_pred_raw": 1.4697870718397748, + "layer": 11, + "head": 5, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.050474359770305455, + "spec_norm": 18.683054315466933, + "frob": 97.40623997433379, + "SR": 27.181774531023397, + "cos_in": -0.5880383849143982, + "cos_out": 0.03720882907509804, + "align_raw": 0.10538871586322784, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": -0.017539766052616347, + "R_pred_norm": -4.469147008890773, + "R_pred_raw": -0.6775242960015736, + "layer": 11, + "head": 6, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.18513384321704507, + "spec_norm": 16.053417680926426, + "frob": 76.0262341645961, + "SR": 22.42807228285055, + "cos_in": 0.009363888762891293, + "cos_out": -0.025468189269304276, + "align_raw": -0.07213358581066132, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": 0.005533401839942561, + "R_pred_norm": -0.15351891862976746, + "R_pred_raw": -0.027085312477742613, + "layer": 11, + "head": 7, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.029370348900556564, + "spec_norm": 334.85910049974757, + "frob": 338.5618019653133, + "SR": 1.022237249959398, + "cos_in": 0.4618133008480072, + "cos_out": 0.023359255865216255, + "align_raw": 0.0661589503288269, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": -0.05571781558877605, + "R_pred_norm": 22.980047315016744, + "R_pred_raw": 0.1943651575230762, + "layer": 11, + "head": 8, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.051125537836924195, + "spec_norm": 18.626947288356273, + "frob": 90.40253945893888, + "SR": 23.55471692205203, + "cos_in": -0.35444381833076477, + "cos_out": -0.09464764595031738, + "align_raw": -0.26806503534317017, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": 0.04712185287739756, + "R_pred_norm": 6.919760516821421, + "R_pred_raw": 1.0521550237695962, + "layer": 11, + "head": 9, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.020433248952031136, + "spec_norm": 14.325500713871225, + "frob": 62.8168530829904, + "SR": 19.22793877092154, + "cos_in": -0.06283044070005417, + "cos_out": 0.002191105391830206, + "align_raw": 0.006205707788467407, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": -0.00010005303510561488, + "R_pred_norm": -0.008728404735547233, + "R_pred_raw": -0.0017256516058492958, + "layer": 11, + "head": 10, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.02439418830908835, + "spec_norm": 62.309257767330415, + "frob": 95.02363501222999, + "SR": 2.3257237279019654, + "cos_in": 0.29600760340690613, + "cos_out": 0.06152227148413658, + "align_raw": 0.1742546111345291, + "norm_xA": 264.2615966796875, + "scale": 216.59791204072337, + "R_obs": 0.03561028218989451, + "R_pred_norm": 5.995541925779821, + "R_pred_raw": 0.27253845133140914, + "layer": 11, + "head": 11, + "word_A": "water", + "word_B": " is", + "gap": 1.2200560569763184 + }, + { + "delta_attn": 0.0005474124664033297, + "spec_norm": 6.036941609910174, + "frob": 31.60514503796959, + "SR": 27.408270051027085, + "cos_in": -0.2824779152870178, + "cos_out": -0.06448829174041748, + "align_raw": -0.17908692359924316, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": -0.028809310349207844, + "R_pred_norm": 0.0076891244363503555, + "R_pred_raw": 0.0035370639934042404, + "layer": 9, + "head": 0, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 7.70940077927662e-05, + "spec_norm": 10.93353757745038, + "frob": 48.79774054627811, + "SR": 19.919481210910376, + "cos_in": 0.35373982787132263, + "cos_out": 0.07708609104156494, + "align_raw": 0.21406838297843933, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": -0.0062244971770496985, + "R_pred_norm": 0.0029357641800976005, + "R_pred_raw": 0.0007456533446986789, + "layer": 9, + "head": 1, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0006030274744261988, + "spec_norm": 5.876704192031835, + "frob": 34.24818760886753, + "SR": 33.963115826977834, + "cos_in": 0.14978712797164917, + "cos_out": 0.040961381047964096, + "align_raw": 0.11375357955694199, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": 0.03940671043596172, + "R_pred_norm": 0.002777153422722413, + "R_pred_raw": 0.00131237069174668, + "layer": 9, + "head": 2, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.00013353103349800222, + "spec_norm": 7.177969011896205, + "frob": 22.909361998700444, + "SR": 10.186449376848554, + "cos_in": 0.3847108483314514, + "cos_out": 0.029380034655332565, + "align_raw": 0.08159089833498001, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": 0.04203392984887714, + "R_pred_norm": 0.0013837269017926686, + "R_pred_raw": 0.0005353505046331413, + "layer": 9, + "head": 3, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0036515085375867784, + "spec_norm": 10.769035341774249, + "frob": 47.84954141139965, + "SR": 19.74249129875483, + "cos_in": -0.2889537811279297, + "cos_out": -0.03725758567452431, + "align_raw": -0.10346649587154388, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": -0.018192941169219923, + "R_pred_norm": 0.05407195493145141, + "R_pred_raw": 0.013943772916723043, + "layer": 9, + "head": 4, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.005577110918238759, + "spec_norm": 13.503818678302977, + "frob": 53.306145668352585, + "SR": 15.582651852687384, + "cos_in": 0.3830450475215912, + "cos_out": -0.016471577808260918, + "align_raw": -0.045743055641651154, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": 0.006842046545669207, + "R_pred_norm": -0.0606919081178518, + "R_pred_raw": -0.012481424782268744, + "layer": 9, + "head": 5, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 4.669881673180498e-05, + "spec_norm": 8.568407234665175, + "frob": 51.29468564471981, + "SR": 35.83806450218393, + "cos_in": 0.16601601243019104, + "cos_out": 0.01303497701883316, + "align_raw": 0.03619924560189247, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": -0.00961153415783997, + "R_pred_norm": 0.00011059769655982695, + "R_pred_raw": 3.5845548705209104e-05, + "layer": 9, + "head": 6, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.00036270701139073935, + "spec_norm": 9.229841381292971, + "frob": 40.66509439732988, + "SR": 19.411321133223833, + "cos_in": 0.0402650460600853, + "cos_out": -0.020324161276221275, + "align_raw": -0.05644100159406662, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": 0.003901310175135526, + "R_pred_norm": -0.00034992170300693717, + "R_pred_raw": -0.00010528311391111484, + "layer": 9, + "head": 7, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0017887169524328783, + "spec_norm": 9.276381529047894, + "frob": 39.05353959730315, + "SR": 17.724075819383255, + "cos_in": -0.3276630938053131, + "cos_out": -0.08583534508943558, + "align_raw": -0.23837193846702576, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": 0.020468178518792478, + "R_pred_norm": 0.05960652020854305, + "R_pred_raw": 0.017844489866778286, + "layer": 9, + "head": 8, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 9.943493932951242e-05, + "spec_norm": 8.40992628067492, + "frob": 51.80072336759642, + "SR": 37.93912156776617, + "cos_in": -0.0525849424302578, + "cos_out": 0.0019592484459280968, + "align_raw": 0.005440987646579742, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": -0.004041470697364868, + "R_pred_norm": -1.1004288655606357e-05, + "R_pred_raw": -3.633774843131306e-06, + "layer": 9, + "head": 9, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 2.7909966036077094e-06, + "spec_norm": 6.252881415873531, + "frob": 31.1179750167106, + "SR": 24.766365082827907, + "cos_in": 0.295616090297699, + "cos_out": 0.0451798215508461, + "align_raw": 0.12546712160110474, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": -0.001937482106155231, + "R_pred_norm": 2.9770898275977867e-05, + "R_pred_raw": 1.3222000517768282e-05, + "layer": 9, + "head": 10, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.001169293827842921, + "spec_norm": 8.934042094947666, + "frob": 40.733481961825646, + "SR": 20.787730739716768, + "cos_in": 0.27468785643577576, + "cos_out": 0.057052865624427795, + "align_raw": 0.15843690931797028, + "norm_xA": 115.58617401123047, + "scale": 127.7261607616872, + "R_obs": 0.08787537824523257, + "R_pred_norm": 0.020910693338165223, + "R_pred_raw": 0.006499790183521473, + "layer": 9, + "head": 11, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0006143329810583964, + "spec_norm": 9.550612297201935, + "frob": 59.05110670533618, + "SR": 38.22906285048873, + "cos_in": -0.058642301708459854, + "cos_out": -0.034428223967552185, + "align_raw": -0.09560833871364594, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": -0.006705047538978872, + "R_pred_norm": 0.0018217055042050992, + "R_pred_raw": 0.0005296977248143266, + "layer": 10, + "head": 0, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 7.987375056472956e-06, + "spec_norm": 8.474343117432328, + "frob": 50.37616313793658, + "SR": 35.337684185342425, + "cos_in": 0.2232438325881958, + "cos_out": 0.06670065224170685, + "align_raw": 0.18523254990577698, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": 0.014843315455641245, + "R_pred_norm": 0.0001550019523610879, + "R_pred_raw": 5.079469405090475e-05, + "layer": 10, + "head": 1, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 2.3009091819403693e-05, + "spec_norm": 9.941501795341454, + "frob": 55.272725812556054, + "SR": 30.911335920222314, + "cos_in": -0.3605244755744934, + "cos_out": -0.04439253732562065, + "align_raw": -0.12327998131513596, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": -0.0025286855119496746, + "R_pred_norm": 0.0005630068655435378, + "R_pred_raw": 0.00015726942065219786, + "layer": 10, + "head": 2, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.00014599251153413206, + "spec_norm": 9.503399692367497, + "frob": 41.48036998087167, + "SR": 19.051415661602288, + "cos_in": -0.16021420061588287, + "cos_out": -0.05660777539014816, + "align_raw": -0.15720270574092865, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": 0.0025813774375998033, + "R_pred_norm": 0.0019351033569993074, + "R_pred_raw": 0.0005654693890630023, + "layer": 10, + "head": 3, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0007109994767233729, + "spec_norm": 19.162915910461177, + "frob": 41.36431396417161, + "SR": 4.6593835707170745, + "cos_in": -0.5241813063621521, + "cos_out": 0.13066905736923218, + "align_raw": 0.3628663122653961, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": 0.047089720115006985, + "R_pred_norm": -0.1435167721494085, + "R_pred_raw": -0.0207976831319357, + "layer": 10, + "head": 4, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 6.522015337395715e-05, + "spec_norm": 10.470172842324263, + "frob": 56.42047950711249, + "SR": 29.03794266776303, + "cos_in": 0.3298070728778839, + "cos_out": 0.10401926934719086, + "align_raw": 0.2888670563697815, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": -0.013591355302194187, + "R_pred_norm": 0.0036026902723387817, + "R_pred_raw": 0.0009555585280887191, + "layer": 10, + "head": 5, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0003676093037938699, + "spec_norm": 9.375325108295202, + "frob": 56.18359795961362, + "SR": 35.91256474485841, + "cos_in": -0.07493923604488373, + "cos_out": 0.019917486235499382, + "align_raw": 0.05531253665685654, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": 0.011448374686003454, + "R_pred_norm": -0.0007911049725348527, + "R_pred_raw": -0.0002343348211864342, + "layer": 10, + "head": 6, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 8.720940968487412e-05, + "spec_norm": 11.000900231717544, + "frob": 46.6323301878643, + "SR": 17.968746540541265, + "cos_in": -0.4639929533004761, + "cos_out": 0.025874411687254906, + "align_raw": 0.0718531534075737, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": 0.009196848702973456, + "R_pred_norm": -0.0017712951674505374, + "R_pred_raw": -0.00044713437651096045, + "layer": 10, + "head": 7, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.00014145417117106263, + "spec_norm": 15.539008982908092, + "frob": 50.32241507503763, + "SR": 10.487604850112564, + "cos_in": -0.5128980875015259, + "cos_out": -0.083228699862957, + "align_raw": -0.2311313897371292, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": -0.09115018142438806, + "R_pred_norm": 0.01442981497160288, + "R_pred_raw": 0.0025788332714267097, + "layer": 10, + "head": 8, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 7.599186483275844e-05, + "spec_norm": 9.145337160130344, + "frob": 32.93676432705462, + "SR": 12.970670360230072, + "cos_in": -0.05850332975387573, + "cos_out": -0.0036950951907783747, + "align_raw": -0.010261517018079758, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": -0.010735452931957214, + "R_pred_norm": 2.310417871739079e-05, + "R_pred_raw": 7.0157927767260325e-06, + "layer": 10, + "head": 9, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 8.872277976479381e-05, + "spec_norm": 39.854820765890416, + "frob": 74.28405022067321, + "SR": 3.4739969204351477, + "cos_in": -0.5018484592437744, + "cos_out": -0.10618343204259872, + "align_raw": -0.29488176107406616, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": -0.006848369576747222, + "R_pred_norm": 0.02897762915645424, + "R_pred_raw": 0.002019171199124478, + "layer": 10, + "head": 10, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.00031853355358180124, + "spec_norm": 7.553501039928774, + "frob": 30.829615793274776, + "SR": 16.658643653607324, + "cos_in": -0.467673122882843, + "cos_out": -0.03495856001973152, + "align_raw": -0.09707991778850555, + "norm_xA": 139.16932678222656, + "scale": 153.7862461297089, + "R_obs": 0.017403616122980997, + "R_pred_norm": 0.006049464369306874, + "R_pred_raw": 0.002224049732537985, + "layer": 10, + "head": 11, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.01704172370955348, + "spec_norm": 61.65889700666178, + "frob": 86.9711978635025, + "SR": 1.9895708090537703, + "cos_in": -0.43416666984558105, + "cos_out": -0.09899403899908066, + "align_raw": -0.27490875124931335, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": -0.30659850778040365, + "R_pred_norm": 10.092049286199893, + "R_pred_raw": 0.4545306064287167, + "layer": 11, + "head": 0, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0013703759323107079, + "spec_norm": 18.265131282089193, + "frob": 77.23712031080814, + "SR": 17.881607190408033, + "cos_in": 0.38727596402168274, + "cos_out": 0.09255477786064148, + "align_raw": 0.25702792406082153, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": 0.022406714463460712, + "R_pred_norm": 0.20048711849828338, + "R_pred_raw": 0.03048211812015794, + "layer": 11, + "head": 1, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 9.568832319928333e-05, + "spec_norm": 18.919704826262237, + "frob": 106.23267321956556, + "SR": 31.527352839346545, + "cos_in": 0.15510858595371246, + "cos_out": -0.014240540564060211, + "align_raw": -0.039546869695186615, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": -0.00908303414356918, + "R_pred_norm": -0.0008935938711903202, + "R_pred_raw": -0.00013116303660744616, + "layer": 11, + "head": 2, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0020272053370717913, + "spec_norm": 13.58022967182268, + "frob": 68.92652762860973, + "SR": 25.760754015986006, + "cos_in": -0.1744176149368286, + "cos_out": -0.03721161559224129, + "align_raw": -0.10333812236785889, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": 0.04288806596366573, + "R_pred_norm": 0.039928067588788516, + "R_pred_raw": 0.008164944132047752, + "layer": 11, + "head": 3, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0005724069196730852, + "spec_norm": 48.044989800413894, + "frob": 81.35082986381424, + "SR": 2.8670004694928535, + "cos_in": -0.620450496673584, + "cos_out": 0.05945584177970886, + "align_raw": 0.16511403024196625, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": -0.0290769853315105, + "R_pred_norm": -0.22670401744072424, + "R_pred_raw": -0.01310389959402984, + "layer": 11, + "head": 4, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 6.0522348576341756e-05, + "spec_norm": 19.05751647844419, + "frob": 80.7581869882243, + "SR": 17.95727829022412, + "cos_in": -0.36069679260253906, + "cos_out": -0.07472896575927734, + "align_raw": -0.20752277970314026, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": 0.0024148709525453967, + "R_pred_norm": 0.00694732947304471, + "R_pred_raw": 0.0010123446588821505, + "layer": 11, + "head": 5, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0022600325755774975, + "spec_norm": 18.683054315466933, + "frob": 97.40623997433379, + "SR": 27.181774531023397, + "cos_in": -0.4964818060398102, + "cos_out": -0.026155080646276474, + "align_raw": -0.07263482362031937, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": 0.011706038202432583, + "R_pred_norm": 0.12252551701663228, + "R_pred_raw": 0.01821241300029433, + "layer": 11, + "head": 6, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.00015782015543663874, + "spec_norm": 16.053417680926426, + "frob": 76.0262341645961, + "SR": 22.42807228285055, + "cos_in": -0.004176364745944738, + "cos_out": -0.051577091217041016, + "align_raw": -0.14323100447654724, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": -0.0005843534554599268, + "R_pred_norm": 0.00012195224673915046, + "R_pred_raw": 2.109611539362264e-05, + "layer": 11, + "head": 7, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.027377542108297348, + "spec_norm": 334.85910049974757, + "frob": 338.5618019653133, + "SR": 1.022237249959398, + "cos_in": 0.4448018968105316, + "cos_out": 0.04846421629190445, + "align_raw": 0.13458341360092163, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": -0.026934531634576268, + "R_pred_norm": 44.16202238399123, + "R_pred_raw": 0.36623276231935753, + "layer": 11, + "head": 8, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 8.286567754112184e-05, + "spec_norm": 18.626947288356273, + "frob": 90.40253945893888, + "SR": 23.55471692205203, + "cos_in": -0.3171490430831909, + "cos_out": -0.12495534121990204, + "align_raw": -0.3469971716403961, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": 0.00041837388966202156, + "R_pred_norm": 0.013669076315948004, + "R_pred_raw": 0.0020378330092408483, + "layer": 11, + "head": 9, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.00033217956661246717, + "spec_norm": 14.325500713871225, + "frob": 62.8168530829904, + "SR": 19.22793877092154, + "cos_in": -0.052589163184165955, + "cos_out": -0.007701172027736902, + "align_raw": -0.021385807543992996, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": 0.0001027492550177509, + "R_pred_norm": 0.00043066579166601096, + "R_pred_raw": 8.348325915241447e-05, + "layer": 11, + "head": 10, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.0002744180164881982, + "spec_norm": 62.309257767330415, + "frob": 95.02363501222999, + "SR": 2.3257237279019654, + "cos_in": 0.25930315256118774, + "cos_out": 0.04069748893380165, + "align_raw": 0.11302129179239273, + "norm_xA": 202.2230224609375, + "scale": 223.462455587901, + "R_obs": -0.0017119606643726801, + "R_pred_norm": 0.04032228884245931, + "R_pred_raw": 0.0017971538244712006, + "layer": 11, + "head": 11, + "word_A": "Romeo", + "word_B": " and", + "gap": -0.9049530029296875 + }, + { + "delta_attn": 0.005341159645467997, + "spec_norm": 6.036941609910174, + "frob": 31.60514503796959, + "SR": 27.408270051027085, + "cos_in": -0.40814709663391113, + "cos_out": -0.03180084750056267, + "align_raw": -0.10051608085632324, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": 0.003776802375038508, + "R_pred_norm": 0.06906043119775514, + "R_pred_raw": 0.03615839653226278, + "layer": 9, + "head": 0, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.01880625542253256, + "spec_norm": 10.93353757745038, + "frob": 48.79774054627811, + "SR": 19.919481210910376, + "cos_in": 0.5177115201950073, + "cos_out": 0.0219191312789917, + "align_raw": 0.06928092986345291, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": 0.10422950833262129, + "R_pred_norm": 0.38503086468370024, + "R_pred_raw": 0.11130770105885714, + "layer": 9, + "head": 1, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.006288922042585909, + "spec_norm": 5.876704192031835, + "frob": 34.24818760886753, + "SR": 33.963115826977834, + "cos_in": 0.18265889585018158, + "cos_out": -0.008923848159611225, + "align_raw": -0.028206974267959595, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": 0.02348929771410687, + "R_pred_norm": -0.0099408698472624, + "R_pred_raw": -0.0053468116696531245, + "layer": 9, + "head": 2, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0007387566381567012, + "spec_norm": 7.177969011896205, + "frob": 22.909361998700444, + "SR": 10.186449376848554, + "cos_in": 0.5647002458572388, + "cos_out": 0.029425954446196556, + "align_raw": 0.09301089495420456, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": -0.2449130203056675, + "R_pred_norm": 0.014540282891475015, + "R_pred_raw": 0.006402867900974491, + "layer": 9, + "head": 3, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0007727956690359861, + "spec_norm": 10.769035341774249, + "frob": 47.84954141139965, + "SR": 19.74249129875483, + "cos_in": -0.4567500650882721, + "cos_out": -0.000604473112616688, + "align_raw": -0.0019106268882751465, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": -0.03747260687535787, + "R_pred_norm": 0.00037915643670775423, + "R_pred_raw": 0.00011128599887867913, + "layer": 9, + "head": 4, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0033004189608618617, + "spec_norm": 13.503818678302977, + "frob": 53.306145668352585, + "SR": 15.582651852687384, + "cos_in": 0.5548374652862549, + "cos_out": -0.0033612039405852556, + "align_raw": -0.01062425971031189, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": 0.06089536266959475, + "R_pred_norm": -0.013715374165378854, + "R_pred_raw": -0.0032103684343043827, + "layer": 9, + "head": 5, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0011818700877483934, + "spec_norm": 8.568407234665175, + "frob": 51.29468564471981, + "SR": 35.83806450218393, + "cos_in": 0.25442805886268616, + "cos_out": 0.042846761643886566, + "align_raw": 0.1354319453239441, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": 0.03466102795337857, + "R_pred_norm": 0.01821688824209945, + "R_pred_raw": 0.00672012277963262, + "layer": 9, + "head": 6, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.004908852431981359, + "spec_norm": 9.229841381292971, + "frob": 40.66509439732988, + "SR": 19.411321133223833, + "cos_in": 0.02919868566095829, + "cos_out": -0.12851247191429138, + "align_raw": -0.40620121359825134, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": -0.12851292170583595, + "R_pred_norm": -0.02805460380166563, + "R_pred_raw": -0.009607399311124164, + "layer": 9, + "head": 7, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0035800995538011193, + "spec_norm": 9.276381529047894, + "frob": 39.05353959730315, + "SR": 17.724075819383255, + "cos_in": -0.5122240781784058, + "cos_out": -0.010057475417852402, + "align_raw": -0.03179009258747101, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": -0.0026243840704769034, + "R_pred_norm": 0.02823215442201808, + "R_pred_raw": 0.009619848474875892, + "layer": 9, + "head": 8, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.002112237096298486, + "spec_norm": 8.40992628067492, + "frob": 51.80072336759642, + "SR": 37.93912156776617, + "cos_in": -0.09374354034662247, + "cos_out": -0.009408509358763695, + "align_raw": -0.029738761484622955, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": 0.06036010239108009, + "R_pred_norm": 0.002585344208823524, + "R_pred_raw": 0.0009716911341144523, + "layer": 9, + "head": 9, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0002909071579324518, + "spec_norm": 6.252881415873531, + "frob": 31.1179750167106, + "SR": 24.766365082827907, + "cos_in": 0.42641377449035645, + "cos_out": 0.060160230845212936, + "align_raw": 0.19015544652938843, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": -0.02605079601415719, + "R_pred_norm": 0.007700105310857338, + "R_pred_raw": 0.0038923846171108954, + "layer": 9, + "head": 10, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0031360617140308022, + "spec_norm": 8.934042094947666, + "frob": 40.733481961825646, + "SR": 20.787730739716768, + "cos_in": 0.43734291195869446, + "cos_out": 0.03530854731798172, + "align_raw": 0.11160215735435486, + "norm_xA": 107.60625457763672, + "scale": 165.01421038463508, + "R_obs": 0.0492037279794707, + "R_pred_norm": 0.07139304125612246, + "R_pred_raw": 0.025258097089259024, + "layer": 9, + "head": 11, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.009045091108419001, + "spec_norm": 9.550612297201935, + "frob": 59.05110670533618, + "SR": 38.22906285048873, + "cos_in": -0.047479405999183655, + "cos_out": 0.009788802824914455, + "align_raw": 0.030940309166908264, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": 0.06267444498875616, + "R_pred_norm": -0.008297633007892108, + "R_pred_raw": -0.002746110923302487, + "layer": 10, + "head": 0, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0015356993535533547, + "spec_norm": 8.474343117432328, + "frob": 50.37616313793658, + "SR": 35.337684185342425, + "cos_in": 0.34233495593070984, + "cos_out": 0.05018279328942299, + "align_raw": 0.15861926972866058, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": 0.09673440197363876, + "R_pred_norm": 0.04620548448377326, + "R_pred_raw": 0.017234099612073076, + "layer": 10, + "head": 1, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.00756003987044096, + "spec_norm": 9.941501795341454, + "frob": 55.272725812556054, + "SR": 30.911335920222314, + "cos_in": -0.47743624448776245, + "cos_out": 0.007682099472731352, + "align_raw": 0.024281548336148262, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": -0.05573068596583928, + "R_pred_norm": -0.05697004950011868, + "R_pred_raw": -0.01811302771452882, + "layer": 10, + "head": 2, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.018002033233642578, + "spec_norm": 9.503399692367497, + "frob": 41.48036998087167, + "SR": 19.051415661602288, + "cos_in": -0.2399081587791443, + "cos_out": -0.030150244012475014, + "align_raw": -0.09529906511306763, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": -0.005561003303420688, + "R_pred_norm": 0.25574762373358184, + "R_pred_raw": 0.08506098762955032, + "layer": 10, + "head": 3, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.003635316388681531, + "spec_norm": 19.162915910461177, + "frob": 41.36431396417161, + "SR": 4.6593835707170745, + "cos_in": -0.7305701375007629, + "cos_out": 0.06652700901031494, + "align_raw": 0.21027407050132751, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": -0.04436664226584442, + "R_pred_norm": -0.6997435866633631, + "R_pred_raw": -0.11541574422428703, + "layer": 10, + "head": 4, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0017603959740881692, + "spec_norm": 10.470172842324263, + "frob": 56.42047950711249, + "SR": 29.03794266776303, + "cos_in": 0.4717004895210266, + "cos_out": 0.08819513767957687, + "align_raw": 0.278767853975296, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": -0.09003268004387537, + "R_pred_norm": 0.15847121628296396, + "R_pred_raw": 0.04784037655364695, + "layer": 10, + "head": 5, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.014926415053196251, + "spec_norm": 9.375325108295202, + "frob": 56.18359795961362, + "SR": 35.91256474485841, + "cos_in": -0.08860839158296585, + "cos_out": 0.019284158945083618, + "align_raw": 0.06095419079065323, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": 0.01709542356747827, + "R_pred_norm": -0.049418847167797, + "R_pred_raw": -0.016661309850791013, + "layer": 10, + "head": 6, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0034446179633960128, + "spec_norm": 11.000900231717544, + "frob": 46.6323301878643, + "SR": 17.968746540541265, + "cos_in": -0.6168413162231445, + "cos_out": 0.003989251796156168, + "align_raw": 0.012608999386429787, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": -0.5017158386012917, + "R_pred_norm": -0.019271228304054804, + "R_pred_raw": -0.005536946723836368, + "layer": 10, + "head": 7, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.052217843011021614, + "spec_norm": 15.539008982908092, + "frob": 50.32241507503763, + "SR": 10.487604850112564, + "cos_in": -0.714381754398346, + "cos_out": -0.054887961596250534, + "align_raw": -0.17349094152450562, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": -0.019340299325729622, + "R_pred_norm": 6.575441318429304, + "R_pred_raw": 1.3375230959910211, + "layer": 10, + "head": 8, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0006899752568187978, + "spec_norm": 9.145337160130344, + "frob": 32.93676432705462, + "SR": 12.970670360230072, + "cos_in": -0.0656173899769783, + "cos_out": -0.005634641274809837, + "align_raw": -0.017810087651014328, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": 0.03855263342094003, + "R_pred_norm": 0.000482162761565419, + "R_pred_raw": 0.00016664555316202578, + "layer": 10, + "head": 9, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.012423197738826275, + "spec_norm": 39.854820765890416, + "frob": 74.28405022067321, + "SR": 3.4739969204351477, + "cos_in": -0.7084402441978455, + "cos_out": -0.0706375241279602, + "align_raw": -0.22327519953250885, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": 0.011498589999512206, + "R_pred_norm": 5.120686108168416, + "R_pred_raw": 0.4061180685430429, + "layer": 10, + "head": 10, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.017936316085979342, + "spec_norm": 7.553501039928774, + "frob": 30.829615793274776, + "SR": 16.658643653607324, + "cos_in": -0.6745137572288513, + "cos_out": -0.007472752593457699, + "align_raw": -0.023619480431079865, + "norm_xA": 134.76942443847656, + "scale": 206.66893615985728, + "R_obs": -0.08498792504186363, + "R_pred_norm": 0.1411328663070033, + "R_pred_raw": 0.059056758331636545, + "layer": 10, + "head": 11, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.07238719845190644, + "spec_norm": 61.65889700666178, + "frob": 86.9711978635025, + "SR": 1.9895708090537703, + "cos_in": -0.4738379120826721, + "cos_out": -0.06714899837970734, + "align_raw": -0.21224276721477509, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": -0.10666231017226371, + "R_pred_norm": 39.94145553678536, + "R_pred_raw": 2.04748861770387, + "layer": 11, + "head": 0, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.050326441414654255, + "spec_norm": 18.265131282089193, + "frob": 77.23712031080814, + "SR": 17.881607190408033, + "cos_in": 0.41985151171684265, + "cos_out": -0.015368040651082993, + "align_raw": -0.04857511445879936, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": 0.001999182515654459, + "R_pred_norm": -1.6681314275225827, + "R_pred_raw": -0.2886707618649201, + "layer": 11, + "head": 1, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.006535360473208129, + "spec_norm": 18.919704826262237, + "frob": 106.23267321956556, + "SR": 31.527352839346545, + "cos_in": 0.18745474517345428, + "cos_out": -0.027276691049337387, + "align_raw": -0.08621665835380554, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": 0.028369525991165344, + "R_pred_norm": -0.17781526808099027, + "R_pred_raw": -0.02970668518404975, + "layer": 11, + "head": 2, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.011604735598666593, + "spec_norm": 13.58022967182268, + "frob": 68.92652762860973, + "SR": 25.760754015986006, + "cos_in": -0.19110190868377686, + "cos_out": 0.009009214118123055, + "align_raw": 0.028476249426603317, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": 0.05128042086332537, + "R_pred_norm": -0.07631173372101761, + "R_pred_raw": -0.017761518195042337, + "layer": 11, + "head": 3, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.016280755400657654, + "spec_norm": 48.044989800413894, + "frob": 81.35082986381424, + "SR": 2.8670004694928535, + "cos_in": -0.6815187931060791, + "cos_out": -0.008778288029134274, + "align_raw": -0.027746815234422684, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": 0.019594767326990687, + "R_pred_norm": 1.316155921154644, + "R_pred_raw": 0.08658896569099914, + "layer": 11, + "head": 4, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.025292276870459318, + "spec_norm": 19.05751647844419, + "frob": 80.7581869882243, + "SR": 17.95727829022412, + "cos_in": -0.4064271152019501, + "cos_out": -0.02347932942211628, + "align_raw": -0.07421237975358963, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": -0.2928553767271631, + "R_pred_norm": 1.2936584128496886, + "R_pred_raw": 0.21455763936044214, + "layer": 11, + "head": 5, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.020683829206973314, + "spec_norm": 18.683054315466933, + "frob": 97.40623997433379, + "SR": 27.181774531023397, + "cos_in": -0.5496162176132202, + "cos_out": -0.0009278978686779737, + "align_raw": -0.002932937815785408, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": 0.024753594191636703, + "R_pred_norm": 0.055428810361842176, + "R_pred_raw": 0.00937757096265549, + "layer": 11, + "head": 6, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.15347926504909992, + "spec_norm": 16.053417680926426, + "frob": 76.0262341645961, + "SR": 22.42807228285055, + "cos_in": 0.021009841933846474, + "cos_out": -0.0062135010957717896, + "align_raw": -0.019639501348137856, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": -0.09023449949315139, + "R_pred_norm": -0.0904635033727411, + "R_pred_raw": -0.01781147865235576, + "layer": 11, + "head": 7, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.035522639751434326, + "spec_norm": 334.85910049974757, + "frob": 338.5618019653133, + "SR": 1.022237249959398, + "cos_in": 0.4870365560054779, + "cos_out": 0.0009865144966170192, + "align_raw": 0.003118082880973816, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": -0.16272131835812084, + "R_pred_norm": 1.6074200867934398, + "R_pred_raw": 0.015172301635424405, + "layer": 11, + "head": 8, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.018703007022850215, + "spec_norm": 18.626947288356273, + "frob": 90.40253945893888, + "SR": 23.55471692205203, + "cos_in": -0.3440695106983185, + "cos_out": -0.03208737075328827, + "align_raw": -0.10141894966363907, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": 0.04270528795876068, + "R_pred_norm": 1.0817588746082327, + "R_pred_raw": 0.183558186078515, + "layer": 11, + "head": 9, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.012342854752205312, + "spec_norm": 14.325500713871225, + "frob": 62.8168530829904, + "SR": 19.22793877092154, + "cos_in": -0.04404546692967415, + "cos_out": 0.012623847462236881, + "align_raw": 0.03990013897418976, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": -0.09135328122283366, + "R_pred_norm": -0.027651290672500974, + "R_pred_raw": -0.006100820778938142, + "layer": 11, + "head": 10, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.008687949215527624, + "spec_norm": 62.309257767330415, + "frob": 95.02363501222999, + "SR": 2.3257237279019654, + "cos_in": 0.29947587847709656, + "cos_out": 0.11596174538135529, + "align_raw": 0.3665398955345154, + "norm_xA": 183.40585327148438, + "scale": 281.2529083583955, + "R_obs": -0.008189043523340997, + "R_pred_norm": 5.287415801820012, + "R_pred_raw": 0.26822385186885, + "layer": 11, + "head": 11, + "word_A": "they", + "word_B": " were", + "gap": -0.6521029472351074 + }, + { + "delta_attn": 0.0017557360260980204, + "spec_norm": 6.036941609910174, + "frob": 31.60514503796959, + "SR": 27.408270051027085, + "cos_in": -0.4786236584186554, + "cos_out": -0.04785465449094772, + "align_raw": -0.13553966581821442, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": -0.024582444816958256, + "R_pred_norm": 0.0795286476043597, + "R_pred_raw": 0.037312027385431226, + "layer": 9, + "head": 0, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.053608030546456575, + "spec_norm": 10.93353757745038, + "frob": 48.79774054627811, + "SR": 19.919481210910376, + "cos_in": 0.583210825920105, + "cos_out": 0.07278033345937729, + "align_raw": 0.20613409578800201, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": 0.3077705313860094, + "R_pred_norm": 8.150030364588181, + "R_pred_raw": 2.111224010530351, + "layer": 9, + "head": 1, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.0021164639911148697, + "spec_norm": 5.876704192031835, + "frob": 34.24818760886753, + "SR": 33.963115826977834, + "cos_in": 0.26404836773872375, + "cos_out": 0.008959044702351093, + "align_raw": 0.02537531405687332, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": -0.023680033575551897, + "R_pred_norm": 0.009638711019708509, + "R_pred_raw": 0.004645525533493639, + "layer": 9, + "head": 2, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.001909781670292432, + "spec_norm": 7.177969011896205, + "frob": 22.909361998700444, + "SR": 10.186449376848554, + "cos_in": 0.6670820116996765, + "cos_out": 0.02239791676402092, + "align_raw": 0.06343899667263031, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": -0.1552960776337949, + "R_pred_norm": 0.06709670764660045, + "R_pred_raw": 0.026475752382326252, + "layer": 9, + "head": 3, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.019396483898162842, + "spec_norm": 10.769035341774249, + "frob": 47.84954141139965, + "SR": 19.74249129875483, + "cos_in": -0.5301306247711182, + "cos_out": -0.033774927258491516, + "align_raw": -0.09566183388233185, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": -0.07582288030610938, + "R_pred_norm": 1.2251977275785564, + "R_pred_raw": 0.32223570499300097, + "layer": 9, + "head": 4, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.0104714494664222, + "spec_norm": 13.503818678302977, + "frob": 53.306145668352585, + "SR": 15.582651852687384, + "cos_in": 0.6481311917304993, + "cos_out": -0.0035468144342303276, + "align_raw": -0.010045873001217842, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": 0.25195448960240346, + "R_pred_norm": -0.10648632602932735, + "R_pred_raw": -0.02233502597257606, + "layer": 9, + "head": 5, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.0029270761879161, + "spec_norm": 8.568407234665175, + "frob": 51.29468564471981, + "SR": 35.83806450218393, + "cos_in": 0.2602103054523468, + "cos_out": 0.02167876996099949, + "align_raw": 0.0614020898938179, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": 0.1271904159701065, + "R_pred_norm": 0.046347072771347884, + "R_pred_raw": 0.01532042195191444, + "layer": 9, + "head": 6, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.021303452464053407, + "spec_norm": 9.229841381292971, + "frob": 40.66509439732988, + "SR": 19.411321133223833, + "cos_in": 0.021723803132772446, + "cos_out": 0.019487671554088593, + "align_raw": 0.055195197463035583, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": -0.1791998075170757, + "R_pred_norm": 0.02726899254731558, + "R_pred_raw": 0.008367894491629732, + "layer": 9, + "head": 7, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.0019364391482667997, + "spec_norm": 9.276381529047894, + "frob": 39.05353959730315, + "SR": 17.724075819383255, + "cos_in": -0.5511641502380371, + "cos_out": -0.03539680317044258, + "align_raw": -0.10025643557310104, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": 0.063091255791789, + "R_pred_norm": 0.11480386609756472, + "R_pred_raw": 0.03505307565160732, + "layer": 9, + "head": 8, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.009163236012682319, + "spec_norm": 8.40992628067492, + "frob": 51.80072336759642, + "SR": 37.93912156776617, + "cos_in": -0.06894908100366592, + "cos_out": -0.001957796048372984, + "align_raw": -0.0055451709777116776, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": -0.01909934327410329, + "R_pred_norm": 0.003407736809947275, + "R_pred_raw": 0.0011476814192776358, + "layer": 9, + "head": 9, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.000795787944298354, + "spec_norm": 6.252881415873531, + "frob": 31.1179750167106, + "SR": 24.766365082827907, + "cos_in": 0.5014888644218445, + "cos_out": 0.061899583786726, + "align_raw": 0.17532037198543549, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": -0.008396237550324504, + "R_pred_norm": 0.05060059518374263, + "R_pred_raw": 0.022920288895777824, + "layer": 9, + "head": 10, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.010240088915452361, + "spec_norm": 8.934042094947666, + "frob": 40.733481961825646, + "SR": 20.787730739716768, + "cos_in": 0.5056463479995728, + "cos_out": 0.037499088793992996, + "align_raw": 0.10620839893817902, + "norm_xA": 122.90032196044922, + "scale": 327.58880696148583, + "R_obs": -0.18792904752549663, + "R_pred_norm": 0.5682616737444903, + "R_pred_raw": 0.1801517640082032, + "layer": 9, + "head": 11, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.0025518203983665444, + "spec_norm": 9.550612297201935, + "frob": 59.05110670533618, + "SR": 38.22906285048873, + "cos_in": -0.04364350065588951, + "cos_out": -0.0017648841021582484, + "align_raw": -0.004998695105314255, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": -0.03312484756835562, + "R_pred_norm": 0.0007451565526562649, + "R_pred_raw": 0.00022098193199041055, + "layer": 10, + "head": 0, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.003827428474323824, + "spec_norm": 8.474343117432328, + "frob": 50.37616313793658, + "SR": 35.337684185342425, + "cos_in": 0.35253891348838806, + "cos_out": 0.04222666099667549, + "align_raw": 0.1196005716919899, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": -0.0071366748176009825, + "R_pred_norm": 0.191662596118832, + "R_pred_raw": 0.06405865801458818, + "layer": 10, + "head": 1, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.013799035979900509, + "spec_norm": 9.941501795341454, + "frob": 55.272725812556054, + "SR": 30.911335920222314, + "cos_in": -0.4765062928199768, + "cos_out": -0.04636513069272041, + "align_raw": -0.1313207447528839, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": -0.011602974961688222, + "R_pred_norm": 1.2030711334951414, + "R_pred_raw": 0.34275291730757085, + "layer": 10, + "head": 2, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.02541607606690377, + "spec_norm": 9.503399692367497, + "frob": 41.48036998087167, + "SR": 19.051415661602288, + "cos_in": -0.24830514192581177, + "cos_out": -0.018761340528726578, + "align_raw": -0.05313822627067566, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": 0.0307404482882453, + "R_pred_norm": 0.4466501780681249, + "R_pred_raw": 0.13311644286532426, + "layer": 10, + "head": 3, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.046080597676336765, + "spec_norm": 19.162915910461177, + "frob": 41.36431396417161, + "SR": 4.6593835707170745, + "cos_in": -0.7278405427932739, + "cos_out": 0.10776429623365402, + "align_raw": 0.305216521024704, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": -0.10006215465098353, + "R_pred_norm": -27.492945622624855, + "R_pred_raw": -4.063430349284818, + "layer": 10, + "head": 4, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.005403661358286627, + "spec_norm": 10.470172842324263, + "frob": 56.42047950711249, + "SR": 29.03794266776303, + "cos_in": 0.4860023558139801, + "cos_out": 0.0028477299492806196, + "align_raw": 0.008065704256296158, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": -0.25377583510794616, + "R_pred_norm": 0.031082045232895525, + "R_pred_raw": 0.008408126327273882, + "layer": 10, + "head": 5, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.0017201780283357948, + "spec_norm": 9.375325108295202, + "frob": 56.18359795961362, + "SR": 35.91256474485841, + "cos_in": -0.07643257081508636, + "cos_out": 0.04121171310544014, + "align_raw": 0.1167263612151146, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": -0.06677207685532816, + "R_pred_norm": -0.020164585702701403, + "R_pred_raw": -0.006091878685927739, + "layer": 10, + "head": 6, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.05383436870761216, + "spec_norm": 11.000900231717544, + "frob": 46.6323301878643, + "SR": 17.968746540541265, + "cos_in": -0.6140902638435364, + "cos_out": 0.0064896754920482635, + "align_raw": 0.018380511552095413, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": 0.02512643356529899, + "R_pred_norm": -0.9368573730021877, + "R_pred_raw": -0.24120142326558247, + "layer": 10, + "head": 7, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.11294536478817463, + "spec_norm": 15.539008982908092, + "frob": 50.32241507503763, + "SR": 10.487604850112564, + "cos_in": -0.7017844915390015, + "cos_out": -0.0774403065443039, + "align_raw": -0.2193371206521988, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": 0.09460701514659468, + "R_pred_norm": 37.86107978749504, + "R_pred_raw": 6.901043302969585, + "layer": 10, + "head": 8, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.002018046450757538, + "spec_norm": 9.145337160130344, + "frob": 32.93676432705462, + "SR": 12.970670360230072, + "cos_in": -0.06524499505758286, + "cos_out": 0.0029971201438456774, + "align_raw": 0.008488859981298447, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": -0.06283832644384552, + "R_pred_norm": -0.0014325595042155152, + "R_pred_raw": -0.00044366806210057873, + "layer": 10, + "head": 9, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.0032472828461322933, + "spec_norm": 39.854820765890416, + "frob": 74.28405022067321, + "SR": 3.4739969204351477, + "cos_in": -0.7083244919776917, + "cos_out": -0.0676150992512703, + "align_raw": -0.19151118397712708, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": 0.048936109319193774, + "R_pred_norm": 2.4604093166853516, + "R_pred_raw": 0.17485455644573683, + "layer": 10, + "head": 10, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.0177273903391324, + "spec_norm": 7.553501039928774, + "frob": 30.829615793274776, + "SR": 16.658643653607324, + "cos_in": -0.6844892501831055, + "cos_out": 0.004927599336951971, + "align_raw": 0.013956308364868164, + "norm_xA": 148.92047119140625, + "scale": 396.945091042435, + "R_obs": 0.036156186738431906, + "R_pred_norm": -0.17927740465560632, + "R_pred_raw": -0.06722215510546624, + "layer": 10, + "head": 11, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.1353477295488119, + "spec_norm": 61.65889700666178, + "frob": 86.9711978635025, + "SR": 1.9895708090537703, + "cos_in": -0.4822923541069031, + "cos_out": -0.11062665283679962, + "align_raw": -0.3133275508880615, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": -0.5905760464287741, + "R_pred_norm": 282.5013938773421, + "R_pred_raw": 12.976681789844461, + "layer": 11, + "head": 0, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.01484712038654834, + "spec_norm": 18.265131282089193, + "frob": 77.23712031080814, + "SR": 17.881607190408033, + "cos_in": 0.4272034764289856, + "cos_out": 0.02932600863277912, + "align_raw": 0.08306033909320831, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": 0.02344616925383532, + "R_pred_norm": 2.1555419708208845, + "R_pred_raw": 0.3342523175102083, + "layer": 11, + "head": 1, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.005942889081779867, + "spec_norm": 18.919704826262237, + "frob": 106.23267321956556, + "SR": 31.527352839346545, + "cos_in": 0.13533669710159302, + "cos_out": -0.022341333329677582, + "align_raw": -0.0632781833410263, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": -0.0754822518375222, + "R_pred_norm": -0.21569514103200949, + "R_pred_raw": -0.03229021372560293, + "layer": 11, + "head": 2, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.034663685713894665, + "spec_norm": 13.58022967182268, + "frob": 68.92652762860973, + "SR": 25.760754015986006, + "cos_in": -0.20560993254184723, + "cos_out": -0.017369527369737625, + "align_raw": -0.04919594153761864, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": -0.046974953621320516, + "R_pred_norm": 1.0666395045551018, + "R_pred_raw": 0.2224599446900364, + "layer": 11, + "head": 3, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.053818609565496445, + "spec_norm": 48.044989800413894, + "frob": 81.35082986381424, + "SR": 2.8670004694928535, + "cos_in": -0.6783075928688049, + "cos_out": 0.05836089327931404, + "align_raw": 0.1652991622686386, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": -0.14695449315865416, + "R_pred_norm": -64.94308380854264, + "R_pred_raw": -3.828542950462538, + "layer": 11, + "head": 4, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.029126145876944065, + "spec_norm": 19.05751647844419, + "frob": 80.7581869882243, + "SR": 17.95727829022412, + "cos_in": -0.4193173050880432, + "cos_out": -0.07049424946308136, + "align_raw": -0.19965939223766327, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": 0.12424296291542854, + "R_pred_norm": 10.40997101938954, + "R_pred_raw": 1.5471031494895553, + "layer": 11, + "head": 5, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.006686672801151872, + "spec_norm": 18.683054315466933, + "frob": 97.40623997433379, + "SR": 27.181774531023397, + "cos_in": -0.538162112236023, + "cos_out": 0.03720882907509804, + "align_raw": 0.10538871586322784, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": -0.02682449190124109, + "R_pred_norm": -1.5871602954760478, + "R_pred_raw": -0.24061407237103946, + "layer": 11, + "head": 6, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.04336768388748169, + "spec_norm": 16.053417680926426, + "frob": 76.0262341645961, + "SR": 22.42807228285055, + "cos_in": 0.028578687459230423, + "cos_out": -0.025468189269304276, + "align_raw": -0.07213358581066132, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": -0.01038027127967003, + "R_pred_norm": -0.3214976485904672, + "R_pred_raw": -0.05672176661127045, + "layer": 11, + "head": 7, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.0688817878253758, + "spec_norm": 334.85910049974757, + "frob": 338.5618019653133, + "SR": 1.022237249959398, + "cos_in": 0.4373956024646759, + "cos_out": 0.023359255865216255, + "align_raw": 0.0661589503288269, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": -0.14430064150787042, + "R_pred_norm": 149.52150241459103, + "R_pred_raw": 1.2646523295410297, + "layer": 11, + "head": 8, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.008476463030092418, + "spec_norm": 18.626947288356273, + "frob": 90.40253945893888, + "SR": 23.55471692205203, + "cos_in": -0.36150768399238586, + "cos_out": -0.09464764595031738, + "align_raw": -0.26806503534317017, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": -0.035487639818742124, + "R_pred_norm": 3.4275786306220692, + "R_pred_raw": 0.521166023998601, + "layer": 11, + "head": 9, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.09731541248038411, + "spec_norm": 14.325500713871225, + "frob": 62.8168530829904, + "SR": 19.22793877092154, + "cos_in": -0.07595715671777725, + "cos_out": 0.002191105391830206, + "align_raw": 0.006205707788467407, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": -0.04560227173298409, + "R_pred_norm": -0.14720654667493344, + "R_pred_raw": -0.029103509903313597, + "layer": 11, + "head": 10, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.014546259422786534, + "spec_norm": 62.309257767330415, + "frob": 95.02363501222999, + "SR": 2.3257237279019654, + "cos_in": 0.36847996711730957, + "cos_out": 0.06152227148413658, + "align_raw": 0.1742546111345291, + "norm_xA": 238.02780151367188, + "scale": 634.4592290541149, + "R_obs": -0.015723562630194435, + "R_pred_norm": 13.036285689325052, + "R_pred_raw": 0.5925884860558871, + "layer": 11, + "head": 11, + "word_A": "that", + "word_B": " is", + "gap": -0.3751664161682129 + }, + { + "delta_attn": 0.006290285964496434, + "spec_norm": 6.036941609910174, + "frob": 31.60514503796959, + "SR": 27.408270051027085, + "cos_in": -0.4713445007801056, + "cos_out": 0.04593174159526825, + "align_raw": 0.14513996243476868, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": -0.005339348094388511, + "R_pred_norm": -0.16075724513181808, + "R_pred_raw": -0.08414487571398477, + "layer": 9, + "head": 0, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.05520555563271046, + "spec_norm": 10.93353757745038, + "frob": 48.79774054627811, + "SR": 19.919481210910376, + "cos_in": 0.5796347260475159, + "cos_out": 0.030413590371608734, + "align_raw": 0.09610263258218765, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": 0.14461321809331304, + "R_pred_norm": 2.080642229183862, + "R_pred_raw": 0.6013181313897602, + "layer": 9, + "head": 1, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.0060910353204235435, + "spec_norm": 5.876704192031835, + "frob": 34.24818760886753, + "SR": 33.963115826977834, + "cos_in": 0.24530929327011108, + "cos_out": -0.005019814241677523, + "align_raw": -0.015862412750720978, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": -0.007505337764141826, + "R_pred_norm": -0.00861903073235745, + "R_pred_raw": -0.004634535343198914, + "layer": 9, + "head": 2, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.002260377031234384, + "spec_norm": 7.177969011896205, + "frob": 22.909361998700444, + "SR": 10.186449376848554, + "cos_in": 0.6416245698928833, + "cos_out": 0.028769895434379578, + "align_raw": 0.09091149270534515, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": -0.0426017127520091, + "R_pred_norm": 0.05856438233272505, + "R_pred_raw": 0.02578175596680667, + "layer": 9, + "head": 3, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.031591273844242096, + "spec_norm": 10.769035341774249, + "frob": 47.84954141139965, + "SR": 19.74249129875483, + "cos_in": -0.5151125192642212, + "cos_out": 0.009427418000996113, + "align_raw": 0.029789891093969345, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": 0.0037313761022665583, + "R_pred_norm": -0.3230506670144174, + "R_pred_raw": -0.09479163009836249, + "layer": 9, + "head": 4, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.009575359057635069, + "spec_norm": 13.503818678302977, + "frob": 53.306145668352585, + "SR": 15.582651852687384, + "cos_in": 0.6427080035209656, + "cos_out": -0.014417513273656368, + "align_raw": -0.045558709651231766, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": -0.0011204407579808674, + "R_pred_norm": -0.23428666320829147, + "R_pred_raw": -0.05482416481695595, + "layer": 9, + "head": 5, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.0038043364766053855, + "spec_norm": 8.568407234665175, + "frob": 51.29468564471981, + "SR": 35.83806450218393, + "cos_in": 0.23771613836288452, + "cos_out": 0.07267361134290695, + "align_raw": 0.22964507341384888, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": 0.10761224066363555, + "R_pred_norm": 0.11011494398086563, + "R_pred_raw": 0.04060939822613886, + "layer": 9, + "head": 6, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.0125034618540667, + "spec_norm": 9.229841381292971, + "frob": 40.66509439732988, + "SR": 19.411321133223833, + "cos_in": 0.02566464990377426, + "cos_out": -0.01900683529675007, + "align_raw": -0.060059674084186554, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": -0.03082313435126921, + "R_pred_norm": -0.0110078245716558, + "R_pred_raw": -0.003768603281623041, + "layer": 9, + "head": 7, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.009843369713053107, + "spec_norm": 9.276381529047894, + "frob": 39.05353959730315, + "SR": 17.724075819383255, + "cos_in": -0.5549878478050232, + "cos_out": 0.09128902852535248, + "align_raw": 0.2884686589241028, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": 0.04412451362491364, + "R_pred_norm": -0.9045993930291838, + "R_pred_raw": -0.30814685501044675, + "layer": 9, + "head": 8, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.01128667127341032, + "spec_norm": 8.40992628067492, + "frob": 51.80072336759642, + "SR": 37.93912156776617, + "cos_in": -0.05445338785648346, + "cos_out": -0.057316504418849945, + "align_raw": -0.18111690878868103, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": -0.025006534295748614, + "R_pred_norm": 0.057928795596008345, + "R_pred_raw": 0.02176614968220614, + "layer": 9, + "head": 9, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.001713873165044788, + "spec_norm": 6.252881415873531, + "frob": 31.1179750167106, + "SR": 24.766365082827907, + "cos_in": 0.4927756190299988, + "cos_out": 0.023457355797290802, + "align_raw": 0.07412344217300415, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": 0.07445571133932362, + "R_pred_norm": 0.02422250039823797, + "R_pred_raw": 0.012240955047430124, + "layer": 9, + "head": 10, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.008905998547561467, + "spec_norm": 8.934042094947666, + "frob": 40.733481961825646, + "SR": 20.787730739716768, + "cos_in": 0.5097048282623291, + "cos_out": 0.009714621119201183, + "align_raw": 0.030696995556354523, + "norm_xA": 126.98942565917969, + "scale": 195.53829902939037, + "R_obs": 0.16172967612086872, + "R_pred_norm": 0.07703836590698024, + "R_pred_raw": 0.027247651238095874, + "layer": 9, + "head": 11, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.0014229937369236723, + "spec_norm": 9.550612297201935, + "frob": 59.05110670533618, + "SR": 38.22906285048873, + "cos_in": -0.07038725912570953, + "cos_out": -0.007295666262507439, + "align_raw": -0.023053519427776337, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": 0.056277551256695625, + "R_pred_norm": 0.0016002433765264657, + "R_pred_raw": 0.0005294526171028129, + "layer": 10, + "head": 0, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.01051994925364852, + "spec_norm": 8.474343117432328, + "frob": 50.37616313793658, + "SR": 35.337684185342425, + "cos_in": 0.36241182684898376, + "cos_out": 0.019880060106515884, + "align_raw": 0.06281972676515579, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": 0.01060674127771403, + "R_pred_norm": 0.14727619830103017, + "R_pred_raw": 0.054916755471049794, + "layer": 10, + "head": 1, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.027811480220407248, + "spec_norm": 9.941501795341454, + "frob": 55.272725812556054, + "SR": 30.911335920222314, + "cos_in": -0.49402379989624023, + "cos_out": -0.03221646323800087, + "align_raw": -0.10180088877677917, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": 0.3168894283214145, + "R_pred_norm": 1.0090081020501043, + "R_pred_raw": 0.3207128539836861, + "layer": 10, + "head": 2, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.005762768094427884, + "spec_norm": 9.503399692367497, + "frob": 41.48036998087167, + "SR": 19.051415661602288, + "cos_in": -0.22573836147785187, + "cos_out": -0.01844308152794838, + "align_raw": -0.05827852338552475, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": -0.0855778847350603, + "R_pred_norm": 0.05228075053354301, + "R_pred_raw": 0.017383526014562774, + "layer": 10, + "head": 3, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.024428872857242823, + "spec_norm": 19.162915910461177, + "frob": 41.36431396417161, + "SR": 4.6593835707170745, + "cos_in": -0.7297438383102417, + "cos_out": -0.06972934305667877, + "align_raw": -0.2203335165977478, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": 0.04994698755072509, + "R_pred_norm": 5.4619013801478875, + "R_pred_raw": 0.9006317494242083, + "layer": 10, + "head": 4, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.002795079883071594, + "spec_norm": 10.470172842324263, + "frob": 56.42047950711249, + "SR": 29.03794266776303, + "cos_in": 0.4793429672718048, + "cos_out": 0.05682157352566719, + "align_raw": 0.17955130338668823, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": 0.16909257253045729, + "R_pred_norm": 0.18276814867890198, + "R_pred_raw": 0.055159709146862405, + "layer": 10, + "head": 5, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.0018664907984202728, + "spec_norm": 9.375325108295202, + "frob": 56.18359795961362, + "SR": 35.91256474485841, + "cos_in": -0.08519227802753448, + "cos_out": -0.029263757169246674, + "align_raw": -0.09247198700904846, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": -0.02294186888851257, + "R_pred_norm": 0.010003116497082413, + "R_pred_raw": 0.0033715460459378473, + "layer": 10, + "head": 6, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.006520457565784454, + "spec_norm": 11.000900231717544, + "frob": 46.6323301878643, + "SR": 17.968746540541265, + "cos_in": -0.6070576906204224, + "cos_out": -0.02127288654446602, + "align_raw": -0.06721913069486618, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": 0.008868075671620523, + "R_pred_norm": 0.2124004875155385, + "R_pred_raw": 0.061008978157260016, + "layer": 10, + "head": 7, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.05556811671704054, + "spec_norm": 15.539008982908092, + "frob": 50.32241507503763, + "SR": 10.487604850112564, + "cos_in": -0.7077448964118958, + "cos_out": -0.01160032395273447, + "align_raw": -0.03665616735816002, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": 0.5245439593113443, + "R_pred_norm": 1.6255063889985417, + "R_pred_raw": 0.3305539111589835, + "layer": 10, + "head": 8, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.001872861375886714, + "spec_norm": 9.145337160130344, + "frob": 32.93676432705462, + "SR": 12.970670360230072, + "cos_in": -0.09500637650489807, + "cos_out": 0.01599482074379921, + "align_raw": 0.05054246634244919, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": -0.1580643810593743, + "R_pred_norm": -0.005968020514729257, + "R_pred_raw": -0.0020620901105449186, + "layer": 10, + "head": 9, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.005685409938450903, + "spec_norm": 39.854820765890416, + "frob": 74.28405022067321, + "SR": 3.4739969204351477, + "cos_in": -0.6993618607521057, + "cos_out": 0.12237986922264099, + "align_raw": 0.3867160677909851, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": 0.030350287792855264, + "R_pred_norm": -4.446797932988513, + "R_pred_raw": -0.35257276948297134, + "layer": 10, + "head": 10, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.022094243089668453, + "spec_norm": 7.553501039928774, + "frob": 30.829615793274776, + "SR": 16.658643653607324, + "cos_in": -0.6699373126029968, + "cos_out": -0.04755062982439995, + "align_raw": -0.150253027677536, + "norm_xA": 148.91160583496094, + "scale": 229.2940688530349, + "R_obs": 0.07694623234264675, + "R_pred_norm": 1.2190195840033415, + "R_pred_raw": 0.5099520568015505, + "layer": 10, + "head": 11, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.25547380186617374, + "spec_norm": 61.65889700666178, + "frob": 86.9711978635025, + "SR": 1.9895708090537703, + "cos_in": -0.46652427315711975, + "cos_out": -0.1406073123216629, + "align_raw": -0.4443022906780243, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": 0.015185423169436631, + "R_pred_norm": 348.35322980251755, + "R_pred_raw": 17.852322981752078, + "layer": 11, + "head": 0, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.04871561285108328, + "spec_norm": 18.265131282089193, + "frob": 77.23712031080814, + "SR": 17.881607190408033, + "cos_in": 0.4107557237148285, + "cos_out": -0.00046464617480523884, + "align_raw": -0.0014682328328490257, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": 0.018651006641041936, + "R_pred_norm": -0.057252245605951574, + "R_pred_raw": -0.009904722057247971, + "layer": 11, + "head": 1, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.0016955449536908418, + "spec_norm": 18.919704826262237, + "frob": 106.23267321956556, + "SR": 31.527352839346545, + "cos_in": 0.15661539137363434, + "cos_out": -0.08809012174606323, + "align_raw": -0.2783581614494324, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": 0.020571762226151993, + "R_pred_norm": -0.14920414303600604, + "R_pred_raw": -0.024919726710863368, + "layer": 11, + "head": 2, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.016813170979730785, + "spec_norm": 13.58022967182268, + "frob": 68.92652762860973, + "SR": 25.760754015986006, + "cos_in": -0.19103507697582245, + "cos_out": -0.010230482555925846, + "align_raw": -0.03232728689908981, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": -0.05165775227293703, + "R_pred_norm": 0.15043938468411497, + "R_pred_raw": 0.035004797681215734, + "layer": 11, + "head": 3, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.0349848959594965, + "spec_norm": 48.044989800413894, + "frob": 81.35082986381424, + "SR": 2.8670004694928535, + "cos_in": -0.6627103686332703, + "cos_out": 0.12058574706315994, + "align_raw": 0.38104522228240967, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": -0.03247809730571801, + "R_pred_norm": -45.28391882401588, + "R_pred_raw": -2.9783548525289794, + "layer": 11, + "head": 4, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.014660298358649015, + "spec_norm": 19.05751647844419, + "frob": 80.7581869882243, + "SR": 17.95727829022412, + "cos_in": -0.4008464217185974, + "cos_out": -0.07988130301237106, + "align_raw": -0.2524137794971466, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": 0.02258503126834304, + "R_pred_norm": 3.015975654559956, + "R_pred_raw": 0.5000684404609786, + "layer": 11, + "head": 5, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.0022710408084094524, + "spec_norm": 18.683054315466933, + "frob": 97.40623997433379, + "SR": 27.181774531023397, + "cos_in": -0.5244280695915222, + "cos_out": 0.001371563645079732, + "align_raw": 0.004334069788455963, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": 0.03783800394612452, + "R_pred_norm": -0.010288931495698227, + "R_pred_raw": -0.0017402127022609236, + "layer": 11, + "head": 6, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.042870545759797096, + "spec_norm": 16.053417680926426, + "frob": 76.0262341645961, + "SR": 22.42807228285055, + "cos_in": 0.012811630964279175, + "cos_out": -0.022639833390712738, + "align_raw": -0.07153927534818649, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": 0.043488667165928425, + "R_pred_norm": -0.0672976240628382, + "R_pred_raw": -0.013246572981083036, + "layer": 11, + "head": 7, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.050893415696918964, + "spec_norm": 334.85910049974757, + "frob": 338.5618019653133, + "SR": 1.022237249959398, + "cos_in": 0.4338858425617218, + "cos_out": 0.021654272451996803, + "align_raw": 0.06842345744371414, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": -0.037045031456858904, + "R_pred_norm": 53.98067975720065, + "R_pred_raw": 0.5093750415029142, + "layer": 11, + "head": 8, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.001538395241368562, + "spec_norm": 18.626947288356273, + "frob": 90.40253945893888, + "SR": 23.55471692205203, + "cos_in": -0.3584335148334503, + "cos_out": -0.016018258407711983, + "align_raw": -0.050614796578884125, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": 0.13658363740571228, + "R_pred_norm": 0.0554662860442554, + "R_pred_raw": 0.009409132877112791, + "layer": 11, + "head": 9, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.010567693738266826, + "spec_norm": 14.325500713871225, + "frob": 62.8168530829904, + "SR": 19.22793877092154, + "cos_in": -0.062437742948532104, + "cos_out": 0.026723798364400864, + "align_raw": 0.08444191515445709, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": 0.07341016242755118, + "R_pred_norm": -0.08515908333929664, + "R_pred_raw": -0.018783696208024236, + "layer": 11, + "head": 10, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.009606959181837738, + "spec_norm": 62.309257767330415, + "frob": 95.02363501222999, + "SR": 2.3257237279019654, + "cos_in": 0.3303508162498474, + "cos_out": 0.20833955705165863, + "align_raw": 0.6583479642868042, + "norm_xA": 218.943115234375, + "scale": 337.1285767684765, + "R_obs": -0.00845690475126057, + "R_pred_norm": 13.889327272108648, + "R_pred_raw": 0.704388721979165, + "layer": 11, + "head": 11, + "word_A": "one", + "word_B": " two", + "gap": 0.6494350433349609 + }, + { + "delta_attn": 0.0009639221825636923, + "spec_norm": 6.036941609910174, + "frob": 31.60514503796959, + "SR": 27.408270051027085, + "cos_in": -0.41665905714035034, + "cos_out": 0.012607874348759651, + "align_raw": 0.04370485246181488, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": -0.024167182367373235, + "R_pred_norm": -0.009341321536971205, + "R_pred_raw": -0.005363880976539739, + "layer": 9, + "head": 0, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0009968171652872115, + "spec_norm": 10.93353757745038, + "frob": 48.79774054627811, + "SR": 19.919481210910376, + "cos_in": 0.5289849638938904, + "cos_out": 0.07068726420402527, + "align_raw": 0.24503180384635925, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": 0.1525013434911253, + "R_pred_norm": 0.12453384106618455, + "R_pred_raw": 0.03948280193201548, + "layer": 9, + "head": 1, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0009953335102181882, + "spec_norm": 5.876704192031835, + "frob": 34.24818760886753, + "SR": 33.963115826977834, + "cos_in": 0.22450590133666992, + "cos_out": 0.010197998955845833, + "align_raw": 0.035351693630218506, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": 0.05570415896162861, + "R_pred_norm": 0.004092341365039645, + "R_pred_raw": 0.0024139778698810646, + "layer": 9, + "head": 2, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0003370394506418961, + "spec_norm": 7.177969011896205, + "frob": 22.909361998700444, + "SR": 10.186449376848554, + "cos_in": 0.5694408416748047, + "cos_out": -0.007294472306966782, + "align_raw": -0.025286462157964706, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": 0.2738359632080061, + "R_pred_norm": -0.0030707996482172875, + "R_pred_raw": -0.001483010051052446, + "layer": 9, + "head": 3, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.005602481542155147, + "spec_norm": 10.769035341774249, + "frob": 47.84954141139965, + "SR": 19.74249129875483, + "cos_in": -0.4881868362426758, + "cos_out": 0.04349334165453911, + "align_raw": 0.1507691890001297, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": 0.20692095185271636, + "R_pred_norm": -0.39146482986747244, + "R_pred_raw": -0.1260102201160514, + "layer": 9, + "head": 4, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0015977922012098134, + "spec_norm": 13.503818678302977, + "frob": 53.306145668352585, + "SR": 15.582651852687384, + "cos_in": 0.5676824450492859, + "cos_out": 0.015239519998431206, + "align_raw": 0.05282822623848915, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": 0.19428450016568025, + "R_pred_norm": 0.05704010546591851, + "R_pred_raw": 0.014642608907276368, + "layer": 9, + "head": 5, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0016057711036410183, + "spec_norm": 8.568407234665175, + "frob": 51.29468564471981, + "SR": 35.83806450218393, + "cos_in": 0.2514215111732483, + "cos_out": 0.005317511036992073, + "align_raw": 0.01843327283859253, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": -0.07988645368900228, + "R_pred_norm": 0.005621099105655753, + "R_pred_raw": 0.0022741295136113372, + "layer": 9, + "head": 6, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.005609600761090405, + "spec_norm": 9.229841381292971, + "frob": 40.66509439732988, + "SR": 19.411321133223833, + "cos_in": 0.01670289784669876, + "cos_out": 0.017918527126312256, + "align_raw": 0.06211395189166069, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": 0.29940607632875366, + "R_pred_norm": 0.004735294530082389, + "R_pred_raw": 0.0017784416147766293, + "layer": 9, + "head": 7, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0038319501327350736, + "spec_norm": 9.276381529047894, + "frob": 39.05353959730315, + "SR": 17.724075819383255, + "cos_in": -0.5174171328544617, + "cos_out": 0.02091994881629944, + "align_raw": 0.07251942157745361, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": -0.1923148559122893, + "R_pred_norm": -0.11757812934490491, + "R_pred_raw": -0.043938138925258635, + "layer": 9, + "head": 8, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.002787010103929788, + "spec_norm": 8.40992628067492, + "frob": 51.80072336759642, + "SR": 37.93912156776617, + "cos_in": -0.06574159860610962, + "cos_out": -0.0262791495770216, + "align_raw": -0.09109704196453094, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": -0.08481308304914632, + "R_pred_norm": 0.012373967520857475, + "R_pred_raw": 0.005100464757860443, + "layer": 9, + "head": 9, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.00018378141726316244, + "spec_norm": 6.252881415873531, + "frob": 31.1179750167106, + "SR": 24.766365082827907, + "cos_in": 0.4458235502243042, + "cos_out": -0.026735767722129822, + "align_raw": -0.09267932176589966, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": -0.013052041587048265, + "R_pred_norm": -0.0041856617236218285, + "R_pred_raw": -0.002320460187938586, + "layer": 9, + "head": 10, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0064932373352348804, + "spec_norm": 8.934042094947666, + "frob": 40.733481961825646, + "SR": 20.787730739716768, + "cos_in": 0.45359623432159424, + "cos_out": 0.05333677679300308, + "align_raw": 0.18488872051239014, + "norm_xA": 115.70329284667969, + "scale": 305.58122842269415, + "R_obs": -0.21849450088638805, + "R_pred_norm": 0.42887640282560896, + "R_pred_raw": 0.16640555003920987, + "layer": 9, + "head": 11, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0016360945883207023, + "spec_norm": 9.550612297201935, + "frob": 59.05110670533618, + "SR": 38.22906285048873, + "cos_in": -0.0602334626019001, + "cos_out": 0.02997373789548874, + "align_raw": 0.10390270501375198, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": -0.04616825980134981, + "R_pred_norm": -0.01054349622520916, + "R_pred_raw": -0.0038268318539948177, + "layer": 10, + "head": 0, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0018814930808730423, + "spec_norm": 8.474343117432328, + "frob": 50.37616313793658, + "SR": 35.337684185342425, + "cos_in": 0.289224773645401, + "cos_out": -0.01317602675408125, + "align_raw": -0.04567478224635124, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": 0.20192631687257032, + "R_pred_norm": -0.022708838184408445, + "R_pred_raw": -0.009289255089166771, + "layer": 10, + "head": 1, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.01712642051279545, + "spec_norm": 9.941501795341454, + "frob": 55.272725812556054, + "SR": 30.911335920222314, + "cos_in": -0.4759548306465149, + "cos_out": 0.0014375749742612243, + "align_raw": 0.004983309656381607, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": 0.19180607312560777, + "R_pred_norm": -0.04353926862567445, + "R_pred_raw": -0.015181563654334505, + "layer": 10, + "head": 2, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0021519666915992275, + "spec_norm": 9.503399692367497, + "frob": 41.48036998087167, + "SR": 19.051415661602288, + "cos_in": -0.20663230121135712, + "cos_out": -0.040332332253456116, + "align_raw": -0.1398112177848816, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": -0.004516076913465368, + "R_pred_norm": 0.06369896513697697, + "R_pred_raw": 0.023234967212332733, + "layer": 10, + "head": 3, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0022886897932039574, + "spec_norm": 19.162915910461177, + "frob": 41.36431396417161, + "SR": 4.6593835707170745, + "cos_in": -0.7023671865463257, + "cos_out": 0.10619046539068222, + "align_raw": 0.3680986166000366, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": 0.2698463001678894, + "R_pred_norm": -1.2225441525749907, + "R_pred_raw": -0.22114728911187723, + "layer": 10, + "head": 4, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.000511516059304995, + "spec_norm": 10.470172842324263, + "frob": 56.42047950711249, + "SR": 29.03794266776303, + "cos_in": 0.4814276397228241, + "cos_out": 0.05441956967115402, + "align_raw": 0.18864448368549347, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": -0.07454927188217958, + "R_pred_norm": 0.05244034448702801, + "R_pred_raw": 0.017362037574746463, + "layer": 10, + "head": 5, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.00028819459112128243, + "spec_norm": 9.375325108295202, + "frob": 56.18359795961362, + "SR": 35.91256474485841, + "cos_in": -0.068368099629879, + "cos_out": 0.0212913379073143, + "align_raw": 0.07380686700344086, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": 0.13821764456404104, + "R_pred_norm": -0.0014699241032331408, + "R_pred_raw": -0.0005435035594323951, + "layer": 10, + "head": 6, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.001378041590214707, + "spec_norm": 11.000900231717544, + "frob": 46.6323301878643, + "SR": 17.968746540541265, + "cos_in": -0.5944149494171143, + "cos_out": 0.012570580467581749, + "align_raw": 0.043574802577495575, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": -0.03559212652770726, + "R_pred_norm": -0.042335227210576164, + "R_pred_raw": -0.013339936572198853, + "layer": 10, + "head": 7, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.008455342263914645, + "spec_norm": 15.539008982908092, + "frob": 50.32241507503763, + "SR": 10.487604850112564, + "cos_in": -0.6846903562545776, + "cos_out": -0.03753643110394478, + "align_raw": -0.13011986017227173, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": 0.07781102291560826, + "R_pred_norm": 1.26202508456105, + "R_pred_raw": 0.28153687068809813, + "layer": 10, + "head": 8, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.00016104004566841468, + "spec_norm": 9.145337160130344, + "frob": 32.93676432705462, + "SR": 12.970670360230072, + "cos_in": -0.07205711305141449, + "cos_out": 0.04155265539884567, + "align_raw": 0.14404211938381195, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": 0.048482969608084146, + "R_pred_norm": -0.0016480695483926803, + "R_pred_raw": -0.0006246928485474175, + "layer": 10, + "head": 9, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.004252354498021305, + "spec_norm": 39.854820765890416, + "frob": 74.28405022067321, + "SR": 3.4739969204351477, + "cos_in": -0.6851198077201843, + "cos_out": -0.07472968101501465, + "align_raw": -0.2590532600879669, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": -0.09822782127620519, + "R_pred_norm": 3.2429202083807773, + "R_pred_raw": 0.2820663874043511, + "layer": 10, + "head": 10, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.004333488759584725, + "spec_norm": 7.553501039928774, + "frob": 30.829615793274776, + "SR": 16.658643653607324, + "cos_in": -0.648370087146759, + "cos_out": 0.032436713576316833, + "align_raw": 0.11243913322687149, + "norm_xA": 141.50941467285156, + "scale": 373.7371660321535, + "R_obs": 0.061877558021809095, + "R_pred_norm": -0.25728358111278127, + "R_pred_raw": -0.11807132089362313, + "layer": 10, + "head": 11, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.05052600568160415, + "spec_norm": 61.65889700666178, + "frob": 86.9711978635025, + "SR": 1.9895708090537703, + "cos_in": -0.4635162949562073, + "cos_out": -0.021162908524274826, + "align_raw": -0.07336005568504333, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": -0.1982892755657973, + "R_pred_norm": 16.583273542776375, + "R_pred_raw": 0.9323067114132605, + "layer": 11, + "head": 0, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.010769525892101228, + "spec_norm": 18.265131282089193, + "frob": 77.23712031080814, + "SR": 17.881607190408033, + "cos_in": 0.3996754288673401, + "cos_out": 0.015052787959575653, + "align_raw": 0.05217989534139633, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": -0.02427800634070981, + "R_pred_norm": 0.6421891470693585, + "R_pred_raw": 0.12187830994955685, + "layer": 11, + "head": 1, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0017833156452979892, + "spec_norm": 18.919704826262237, + "frob": 106.23267321956556, + "SR": 31.527352839346545, + "cos_in": 0.1884882003068924, + "cos_out": -0.009266703389585018, + "align_raw": -0.03212296962738037, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": 0.11819628629012234, + "R_pred_norm": -0.031979464826294185, + "R_pred_raw": -0.005859320576599442, + "layer": 11, + "head": 2, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0068666449515149, + "spec_norm": 13.58022967182268, + "frob": 68.92652762860973, + "SR": 25.760754015986006, + "cos_in": -0.19514057040214539, + "cos_out": -0.010444346815347672, + "align_raw": -0.03620496392250061, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": -0.10873343020317901, + "R_pred_norm": 0.10313349777745409, + "R_pred_raw": 0.026325672043076256, + "layer": 11, + "head": 3, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.010750706540420651, + "spec_norm": 48.044989800413894, + "frob": 81.35082986381424, + "SR": 2.8670004694928535, + "cos_in": -0.6757118105888367, + "cos_out": -0.04060366749763489, + "align_raw": -0.1407535970211029, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": 0.09633877627614996, + "R_pred_norm": 7.6900837681435865, + "R_pred_raw": 0.5548520623784889, + "layer": 11, + "head": 4, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0026858042692765594, + "spec_norm": 19.05751647844419, + "frob": 80.7581869882243, + "SR": 17.95727829022412, + "cos_in": -0.37537211179733276, + "cos_out": -0.021682394668459892, + "align_raw": -0.07516035437583923, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": -0.07878073268230329, + "R_pred_norm": 0.22606279770735957, + "R_pred_raw": 0.04111917266436872, + "layer": 11, + "head": 5, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0032021160586737096, + "spec_norm": 18.683054315466933, + "frob": 97.40623997433379, + "SR": 27.181774531023397, + "cos_in": -0.5451005101203918, + "cos_out": 0.022879675030708313, + "align_raw": 0.07931283116340637, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": -0.10339876712302304, + "R_pred_norm": -0.4048839585647836, + "R_pred_raw": -0.07512358331590434, + "layer": 11, + "head": 6, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.01215683575719595, + "spec_norm": 16.053417680926426, + "frob": 76.0262341645961, + "SR": 22.42807228285055, + "cos_in": 0.021762264892458916, + "cos_out": -0.014742452651262283, + "align_raw": -0.05110397934913635, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": -0.013122565933716994, + "R_pred_norm": -0.03397669047840788, + "R_pred_raw": -0.007336662577988193, + "layer": 11, + "head": 7, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.01645258581265807, + "spec_norm": 334.85910049974757, + "frob": 338.5618019653133, + "SR": 1.022237249959398, + "cos_in": 0.44827109575271606, + "cos_out": 0.006861709523946047, + "align_raw": 0.023785235360264778, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": 0.11713086491009118, + "R_pred_norm": 9.195778930260973, + "R_pred_raw": 0.09519223855791384, + "layer": 11, + "head": 8, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0005425397539511323, + "spec_norm": 18.626947288356273, + "frob": 90.40253945893888, + "SR": 23.55471692205203, + "cos_in": -0.32977259159088135, + "cos_out": -0.032838694751262665, + "align_raw": -0.11383131891489029, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": -0.12972197951712597, + "R_pred_norm": 0.05938720429934037, + "R_pred_raw": 0.011051648827750013, + "layer": 11, + "head": 9, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.002520812500733882, + "spec_norm": 14.325500713871225, + "frob": 62.8168530829904, + "SR": 19.22793877092154, + "cos_in": -0.046720631420612335, + "cos_out": -0.019912075251340866, + "align_raw": -0.06902240961790085, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": -0.04192168664122565, + "R_pred_norm": 0.018230339990935946, + "R_pred_raw": 0.004411218289562034, + "layer": 11, + "head": 10, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.0016007233643904328, + "spec_norm": 62.309257767330415, + "frob": 95.02363501222999, + "SR": 2.3257237279019654, + "cos_in": 0.24658027291297913, + "cos_out": 0.09759114682674408, + "align_raw": 0.3383045494556427, + "norm_xA": 205.46517944335938, + "scale": 542.649222745892, + "R_obs": 0.043304467581266076, + "R_pred_norm": 1.3024372172829128, + "R_pred_raw": 0.07246055170327925, + "layer": 11, + "head": 11, + "word_A": "he", + "word_B": " went", + "gap": 0.3786334991455078 + }, + { + "delta_attn": 0.001353324463707395, + "spec_norm": 6.036941609910174, + "frob": 31.60514503796959, + "SR": 27.408270051027085, + "cos_in": -0.40563949942588806, + "cos_out": 0.026689009740948677, + "align_raw": 0.08915266394615173, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": -0.03495950476701941, + "R_pred_norm": -0.013840740269313942, + "R_pred_raw": -0.007658508666173656, + "layer": 9, + "head": 0, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0014639012515544891, + "spec_norm": 10.93353757745038, + "frob": 48.79774054627811, + "SR": 19.919481210910376, + "cos_in": 0.5043257474899292, + "cos_out": -0.012411682866513729, + "align_raw": -0.04145968705415726, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": 0.04074036596532682, + "R_pred_norm": -0.015677687660759665, + "R_pred_raw": -0.004789792011857426, + "layer": 9, + "head": 1, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.004071591189131141, + "spec_norm": 5.876704192031835, + "frob": 34.24818760886753, + "SR": 33.963115826977834, + "cos_in": 0.21871839463710785, + "cos_out": 0.058131590485572815, + "align_raw": 0.19418761134147644, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": 0.04279171901987798, + "R_pred_norm": 0.047606167950422995, + "R_pred_raw": 0.027060680135415783, + "layer": 9, + "head": 2, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.00045235017296363367, + "spec_norm": 7.177969011896205, + "frob": 22.909361998700444, + "SR": 10.186449376848554, + "cos_in": 0.567080557346344, + "cos_out": -0.004873071331530809, + "align_raw": -0.016278374940156937, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": -0.12482730487914215, + "R_pred_norm": -0.0014040791124683536, + "R_pred_raw": -0.0006534288093315364, + "layer": 9, + "head": 3, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.003397287946427241, + "spec_norm": 10.769035341774249, + "frob": 47.84954141139965, + "SR": 19.74249129875483, + "cos_in": -0.46703097224235535, + "cos_out": 0.020313164219260216, + "align_raw": 0.06785494089126587, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": 0.12438552251950237, + "R_pred_norm": -0.05431254273808602, + "R_pred_raw": -0.01684718055388116, + "layer": 9, + "head": 4, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.003328666731249541, + "spec_norm": 13.503818678302977, + "frob": 53.306145668352585, + "SR": 15.582651852687384, + "cos_in": 0.5638983249664307, + "cos_out": 0.014494871720671654, + "align_raw": 0.04841982200741768, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": 0.03314541664407899, + "R_pred_norm": 0.05749231637488802, + "R_pred_raw": 0.01422204411247375, + "layer": 9, + "head": 5, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0007957469206303358, + "spec_norm": 8.568407234665175, + "frob": 51.29468564471981, + "SR": 35.83806450218393, + "cos_in": 0.24710005521774292, + "cos_out": 0.019042132422327995, + "align_raw": 0.06360974907875061, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": -0.03696019187263285, + "R_pred_norm": 0.0050203177080588215, + "R_pred_raw": 0.0019572180140045264, + "layer": 9, + "head": 6, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.008771058593993075, + "spec_norm": 9.229841381292971, + "frob": 40.66509439732988, + "SR": 19.411321133223833, + "cos_in": 0.042565472424030304, + "cos_out": 0.005937041249126196, + "align_raw": 0.019832205027341843, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": -0.10018697100491156, + "R_pred_norm": 0.003201411397672059, + "R_pred_raw": 0.0011586391164256427, + "layer": 9, + "head": 7, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.005203643930144608, + "spec_norm": 9.276381529047894, + "frob": 39.05353959730315, + "SR": 17.724075819383255, + "cos_in": -0.4952763020992279, + "cos_out": -0.00923110917210579, + "align_raw": -0.030836258083581924, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": -0.1802496742427354, + "R_pred_norm": 0.03453467094402207, + "R_pred_raw": 0.012436108557882441, + "layer": 9, + "head": 8, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.002477260131854564, + "spec_norm": 8.40992628067492, + "frob": 51.80072336759642, + "SR": 37.93912156776617, + "cos_in": -0.06897826492786407, + "cos_out": -0.02269790880382061, + "align_raw": -0.07582159340381622, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": -0.006741042718419614, + "R_pred_norm": 0.005104222853813521, + "R_pred_raw": 0.002027423606877612, + "layer": 9, + "head": 9, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.00014239320933029376, + "spec_norm": 6.252881415873531, + "frob": 31.1179750167106, + "SR": 24.766365082827907, + "cos_in": 0.4374661445617676, + "cos_out": 0.02669942006468773, + "align_raw": 0.08918793499469757, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": -0.0018400389748353446, + "R_pred_norm": 0.0016273593391496585, + "R_pred_raw": 0.000869375803170999, + "layer": 9, + "head": 10, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0011243149056099355, + "spec_norm": 8.934042094947666, + "frob": 40.733481961825646, + "SR": 20.787730739716768, + "cos_in": 0.4469711184501648, + "cos_out": 0.018674299120903015, + "align_raw": 0.06237952411174774, + "norm_xA": 120.76344299316406, + "scale": 156.48319788532038, + "R_obs": 0.09289107435788142, + "R_pred_norm": 0.013119807053029552, + "R_pred_raw": 0.004905431323925637, + "layer": 9, + "head": 11, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0007891021086834371, + "spec_norm": 9.550612297201935, + "frob": 59.05110670533618, + "SR": 38.22906285048873, + "cos_in": -0.060192130506038666, + "cos_out": -0.022528858855366707, + "align_raw": -0.07525566965341568, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": -0.004246053671950466, + "R_pred_norm": 0.001944781135576345, + "R_pred_raw": 0.0006802045743542364, + "layer": 10, + "head": 0, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0011674224297166802, + "spec_norm": 8.474343117432328, + "frob": 50.37616313793658, + "SR": 35.337684185342425, + "cos_in": 0.30293166637420654, + "cos_out": 0.03854271396994591, + "align_raw": 0.12875033915042877, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": 0.0030350139168540003, + "R_pred_norm": 0.02198105067174749, + "R_pred_raw": 0.008664599492002045, + "layer": 10, + "head": 1, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.007104058109689504, + "spec_norm": 9.941501795341454, + "frob": 55.272725812556054, + "SR": 30.911335920222314, + "cos_in": -0.4938516318798065, + "cos_out": -0.0067810798063874245, + "align_raw": -0.0226516742259264, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": -0.05052383712971327, + "R_pred_norm": 0.04500712311987702, + "R_pred_raw": 0.015122747802300267, + "layer": 10, + "head": 2, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0013956963375676423, + "spec_norm": 9.503399692367497, + "frob": 41.48036998087167, + "SR": 19.051415661602288, + "cos_in": -0.2301899790763855, + "cos_out": -0.03486088290810585, + "align_raw": -0.11645045131444931, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": 0.026951813325157063, + "R_pred_norm": 0.020254528311051137, + "R_pred_raw": 0.007119441610183985, + "layer": 10, + "head": 3, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0013342489837668836, + "spec_norm": 19.162915910461177, + "frob": 41.36431396417161, + "SR": 4.6593835707170745, + "cos_in": -0.7005294561386108, + "cos_out": 0.037140872329473495, + "align_raw": 0.12406373023986816, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": -0.09690789550476005, + "R_pred_norm": -0.1265914516245349, + "R_pred_raw": -0.02206660501160294, + "layer": 10, + "head": 4, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.001346051724794961, + "spec_norm": 10.470172842324263, + "frob": 56.42047950711249, + "SR": 29.03794266776303, + "cos_in": 0.46578818559646606, + "cos_out": 0.13933731615543365, + "align_raw": 0.4654473066329956, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": -0.01843931602619583, + "R_pred_norm": 0.1740599738826004, + "R_pred_raw": 0.0555326173924807, + "layer": 10, + "head": 5, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0013656788069056347, + "spec_norm": 9.375325108295202, + "frob": 56.18359795961362, + "SR": 35.91256474485841, + "cos_in": -0.09909452497959137, + "cos_out": 0.010663717053830624, + "align_raw": 0.035621851682662964, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": 0.07709256645070717, + "R_pred_norm": -0.002574664139840989, + "R_pred_raw": -0.0009173649128737734, + "layer": 10, + "head": 6, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.002329297858523205, + "spec_norm": 11.000900231717544, + "frob": 46.6323301878643, + "SR": 17.968746540541265, + "cos_in": -0.586347222328186, + "cos_out": -0.01487050112336874, + "align_raw": -0.049672931432724, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": -0.03000289205508887, + "R_pred_norm": 0.04251685518647785, + "R_pred_raw": 0.012910025325961187, + "layer": 10, + "head": 7, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.013990421313792467, + "spec_norm": 15.539008982908092, + "frob": 50.32241507503763, + "SR": 10.487604850112564, + "cos_in": -0.674866259098053, + "cos_out": -0.06467127054929733, + "align_raw": -0.216031014919281, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": -0.1371496343872487, + "R_pred_norm": 1.805554591421136, + "R_pred_raw": 0.3881432329504758, + "layer": 10, + "head": 8, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.00014099247664489667, + "spec_norm": 9.145337160130344, + "frob": 32.93676432705462, + "SR": 12.970670360230072, + "cos_in": -0.058029577136039734, + "cos_out": 0.023337222635746002, + "align_raw": 0.07795679569244385, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": -0.008076893713582143, + "R_pred_norm": -0.0003322933422435131, + "R_pred_raw": -0.0001213742974505561, + "layer": 10, + "head": 9, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.006330114352749661, + "spec_norm": 39.854820765890416, + "frob": 74.28405022067321, + "SR": 3.4739969204351477, + "cos_in": -0.6766566038131714, + "cos_out": 0.04314206913113594, + "align_raw": 0.1441155970096588, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": 0.0004201566497273453, + "R_pred_norm": -1.4014871340220503, + "R_pred_raw": -0.11746764703294593, + "layer": 10, + "head": 10, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0007456282619386911, + "spec_norm": 7.553501039928774, + "frob": 30.829615793274776, + "SR": 16.658643653607324, + "cos_in": -0.640204131603241, + "cos_out": 0.0014895780477672815, + "align_raw": 0.004975743591785431, + "norm_xA": 146.857177734375, + "scale": 190.295011757728, + "R_obs": -0.03306941772067242, + "R_pred_norm": -0.0010220680077415907, + "R_pred_raw": -0.0004519872978542084, + "layer": 10, + "head": 11, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0264206703286618, + "spec_norm": 61.65889700666178, + "frob": 86.9711978635025, + "SR": 1.9895708090537703, + "cos_in": -0.4599459767341614, + "cos_out": -0.052452754229307175, + "align_raw": -0.17521314322948456, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": -0.11140824293976202, + "R_pred_norm": 10.390008819426326, + "R_pred_raw": 0.5628835385745163, + "layer": 11, + "head": 0, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.04866847023367882, + "spec_norm": 18.265131282089193, + "frob": 77.23712031080814, + "SR": 17.881607190408033, + "cos_in": 0.40155917406082153, + "cos_out": 0.020084254443645477, + "align_raw": 0.06708972901105881, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": -0.06928753880459566, + "R_pred_norm": 1.895295674925911, + "R_pred_raw": 0.3466207011340559, + "layer": 11, + "head": 1, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0016060826019383967, + "spec_norm": 18.919704826262237, + "frob": 106.23267321956556, + "SR": 31.527352839346545, + "cos_in": 0.17172296345233917, + "cos_out": -0.019315434619784355, + "align_raw": -0.0645221695303917, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": -0.09591682011334438, + "R_pred_norm": -0.026645035566356896, + "R_pred_raw": -0.004704423654166497, + "layer": 11, + "head": 2, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.004676212658523582, + "spec_norm": 13.58022967182268, + "frob": 68.92652762860973, + "SR": 25.760754015986006, + "cos_in": -0.17732591927051544, + "cos_out": 0.003358784830197692, + "align_raw": 0.011219747364521027, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": 0.040317119928469126, + "R_pred_norm": -0.009999009522304774, + "R_pred_raw": -0.0024595222856324986, + "layer": 11, + "head": 3, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0016991409938782454, + "spec_norm": 48.044989800413894, + "frob": 81.35082986381424, + "SR": 2.8670004694928535, + "cos_in": -0.6561176776885986, + "cos_out": 0.03959695249795914, + "align_raw": 0.1322726458311081, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": -0.05905363499660445, + "R_pred_norm": -0.5606892583053356, + "R_pred_raw": -0.03898364186490986, + "layer": 11, + "head": 4, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.005821182858198881, + "spec_norm": 19.05751647844419, + "frob": 80.7581869882243, + "SR": 17.95727829022412, + "cos_in": -0.3957616984844208, + "cos_out": -0.05275101959705353, + "align_raw": -0.17620836198329926, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": 0.24358396404715021, + "R_pred_norm": 0.6122705579583408, + "R_pred_raw": 0.1073180389137052, + "layer": 11, + "head": 5, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0005523297586478293, + "spec_norm": 18.683054315466933, + "frob": 97.40623997433379, + "SR": 27.181774531023397, + "cos_in": -0.512454628944397, + "cos_out": -0.019498463720083237, + "align_raw": -0.06513404846191406, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": -0.06388111132648644, + "R_pred_norm": 0.02725858736221616, + "R_pred_raw": 0.004873748866746936, + "layer": 11, + "head": 6, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.00522965844720602, + "spec_norm": 16.053417680926426, + "frob": 76.0262341645961, + "SR": 22.42807228285055, + "cos_in": 0.010700412094593048, + "cos_out": -0.011045437306165695, + "align_raw": -0.0368962287902832, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": -0.024652073545325916, + "R_pred_norm": -0.0026231641027850646, + "R_pred_raw": -0.0005458295187080429, + "layer": 11, + "head": 7, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.008972982410341501, + "spec_norm": 334.85910049974757, + "frob": 338.5618019653133, + "SR": 1.022237249959398, + "cos_in": 0.4594619572162628, + "cos_out": 0.012310661375522614, + "align_raw": 0.04112165421247482, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": -0.01847762442661215, + "R_pred_norm": 4.492944742573252, + "R_pred_raw": 0.044818587207814946, + "layer": 11, + "head": 8, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0002818682260112837, + "spec_norm": 18.626947288356273, + "frob": 90.40253945893888, + "SR": 23.55471692205203, + "cos_in": -0.34209227561950684, + "cos_out": -0.04379728436470032, + "align_raw": -0.14629757404327393, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": -0.03358410961658842, + "R_pred_norm": 0.020795983764898118, + "R_pred_raw": 0.003729303473185651, + "layer": 11, + "head": 9, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0006516373250633478, + "spec_norm": 14.325500713871225, + "frob": 62.8168530829904, + "SR": 19.22793877092154, + "cos_in": -0.05084015056490898, + "cos_out": 0.037831779569387436, + "align_raw": 0.1263701319694519, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": -0.04046232112359549, + "R_pred_norm": -0.004746577458118257, + "R_pred_raw": -0.0011067726798153208, + "layer": 11, + "head": 10, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.00239630468422547, + "spec_norm": 62.309257767330415, + "frob": 95.02363501222999, + "SR": 2.3257237279019654, + "cos_in": 0.31499117612838745, + "cos_out": 0.06309183686971664, + "align_raw": 0.2107582837343216, + "norm_xA": 204.0182342529297, + "scale": 264.36332826832387, + "R_obs": 0.02301037322425892, + "R_pred_norm": 0.7844538255979215, + "R_pred_raw": 0.04205583779228, + "layer": 11, + "head": 11, + "word_A": "she", + "word_B": " said", + "gap": -0.7717342376708984 + }, + { + "delta_attn": 0.0007090834697009996, + "spec_norm": 6.036941609910174, + "frob": 31.60514503796959, + "SR": 27.408270051027085, + "cos_in": -0.3604969382286072, + "cos_out": -0.030402449890971184, + "align_raw": -0.09064559638500214, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": -0.09801065258244905, + "R_pred_norm": 0.07016558439917327, + "R_pred_raw": 0.03465335604459505, + "layer": 9, + "head": 0, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0012763704580720514, + "spec_norm": 10.93353757745038, + "frob": 48.79774054627811, + "SR": 19.919481210910376, + "cos_in": 0.4784248471260071, + "cos_out": 0.04397254437208176, + "align_raw": 0.13110318779945374, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": 0.5612200354083071, + "R_pred_norm": 0.43906863746550384, + "R_pred_raw": 0.11973010703164999, + "layer": 9, + "head": 1, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0021947456407360733, + "spec_norm": 5.876704192031835, + "frob": 34.24818760886753, + "SR": 33.963115826977834, + "cos_in": 0.18780292570590973, + "cos_out": 0.020163025707006454, + "align_raw": 0.060117557644844055, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": 0.7087431483169924, + "R_pred_norm": 0.07304251724619855, + "R_pred_raw": 0.03705847346870745, + "layer": 9, + "head": 2, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0002326379553778679, + "spec_norm": 7.177969011896205, + "frob": 22.909361998700444, + "SR": 10.186449376848554, + "cos_in": 0.5224549174308777, + "cos_out": -0.024127110838890076, + "align_raw": -0.07193659245967865, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": 0.9463369993458574, + "R_pred_norm": -0.031480088186155754, + "R_pred_raw": -0.013076120192731263, + "layer": 9, + "head": 3, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.006475602742284536, + "spec_norm": 10.769035341774249, + "frob": 47.84954141139965, + "SR": 19.74249129875483, + "cos_in": -0.41088515520095825, + "cos_out": -0.020730463787913322, + "align_raw": -0.061808641999959946, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": 0.7319095288366075, + "R_pred_norm": 0.8883541148053655, + "R_pred_raw": 0.24595151356474856, + "layer": 9, + "head": 4, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0011864819971378893, + "spec_norm": 13.503818678302977, + "frob": 53.306145668352585, + "SR": 15.582651852687384, + "cos_in": 0.5050241947174072, + "cos_out": -0.033162541687488556, + "align_raw": -0.0988764613866806, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": -0.33804372498216795, + "R_pred_norm": -0.40130760777313834, + "R_pred_raw": -0.08860656518327314, + "layer": 9, + "head": 5, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.00026908084691967815, + "spec_norm": 8.568407234665175, + "frob": 51.29468564471981, + "SR": 35.83806450218393, + "cos_in": 0.23104162514209747, + "cos_out": 0.036207932978868484, + "align_raw": 0.10795633494853973, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": -0.30601649948271614, + "R_pred_norm": 0.028845395438288995, + "R_pred_raw": 0.010037389960809096, + "layer": 9, + "head": 6, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.003108655502728652, + "spec_norm": 9.229841381292971, + "frob": 40.66509439732988, + "SR": 19.411321133223833, + "cos_in": 0.07953521609306335, + "cos_out": -0.06047603860497475, + "align_raw": -0.18031030893325806, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": 0.17390030523279182, + "R_pred_norm": -0.2063998054907689, + "R_pred_raw": -0.06667334865203126, + "layer": 9, + "head": 7, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0006133837159723043, + "spec_norm": 9.276381529047894, + "frob": 39.05353959730315, + "SR": 17.724075819383255, + "cos_in": -0.4576832950115204, + "cos_out": 0.06076553463935852, + "align_raw": 0.1811763197183609, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": 0.0535446425923417, + "R_pred_norm": -0.23666436910209224, + "R_pred_raw": -0.07606736729326163, + "layer": 9, + "head": 8, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0013216649822425097, + "spec_norm": 8.40992628067492, + "frob": 51.80072336759642, + "SR": 37.93912156776617, + "cos_in": -0.05359179526567459, + "cos_out": -0.010398901998996735, + "align_raw": -0.03100493550300598, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": 0.21432791045329663, + "R_pred_norm": 0.009264009927528741, + "R_pred_raw": 0.003284355579983375, + "layer": 9, + "head": 9, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.00012005631901956804, + "spec_norm": 6.252881415873531, + "frob": 31.1179750167106, + "SR": 24.766365082827907, + "cos_in": 0.38440749049186707, + "cos_out": 0.05635504052042961, + "align_raw": 0.16802477836608887, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": -0.055398948634470445, + "R_pred_norm": 0.024321450520861106, + "R_pred_raw": 0.01159711182474515, + "layer": 9, + "head": 10, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0011176696280017495, + "spec_norm": 8.934042094947666, + "frob": 40.733481961825646, + "SR": 20.787730739716768, + "cos_in": 0.42174577713012695, + "cos_out": 0.0016139375511556864, + "align_raw": 0.004811946302652359, + "norm_xA": 114.60527038574219, + "scale": 1495.54556202366, + "R_obs": 3.5202755709114757, + "R_pred_norm": 0.010164793525348263, + "R_pred_raw": 0.0033922247064740257, + "layer": 9, + "head": 11, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.00096995169587899, + "spec_norm": 9.550612297201935, + "frob": 59.05110670533618, + "SR": 38.22906285048873, + "cos_in": -0.05792112648487091, + "cos_out": 0.050884805619716644, + "align_raw": 0.15171357989311218, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": 0.07404779161520816, + "R_pred_norm": -0.05142880299583802, + "R_pred_raw": -0.016055045371464113, + "layer": 10, + "head": 0, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0008418761644861661, + "spec_norm": 8.474343117432328, + "frob": 50.37616313793658, + "SR": 35.337684185342425, + "cos_in": 0.32999664545059204, + "cos_out": 0.02835211530327797, + "align_raw": 0.08453331142663956, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": -0.3881597681476206, + "R_pred_norm": 0.1257329843848794, + "R_pred_raw": 0.04423699271374452, + "layer": 10, + "head": 1, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0005800247308798134, + "spec_norm": 9.941501795341454, + "frob": 55.272725812556054, + "SR": 30.911335920222314, + "cos_in": -0.4463300108909607, + "cos_out": -0.07954004406929016, + "align_raw": -0.23715025186538696, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": 1.513225735440001, + "R_pred_norm": 0.3856029944518331, + "R_pred_raw": 0.11564481494379995, + "layer": 10, + "head": 2, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0020182128937449306, + "spec_norm": 9.503399692367497, + "frob": 41.48036998087167, + "SR": 19.051415661602288, + "cos_in": -0.23036760091781616, + "cos_out": 0.004402306396514177, + "align_raw": 0.013125605881214142, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": 0.10504830403764237, + "R_pred_norm": -0.03663933742516705, + "R_pred_raw": -0.011494966506891248, + "layer": 10, + "head": 3, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0023880260559963062, + "spec_norm": 19.162915910461177, + "frob": 41.36431396417161, + "SR": 4.6593835707170745, + "cos_in": -0.6744274497032166, + "cos_out": 0.03862524405121803, + "align_raw": 0.11515963077545166, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": 0.19345763372410266, + "R_pred_norm": -2.245469208776245, + "R_pred_raw": -0.34936109292612455, + "layer": 10, + "head": 4, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0016209283076022984, + "spec_norm": 10.470172842324263, + "frob": 56.42047950711249, + "SR": 29.03794266776303, + "cos_in": 0.44797852635383606, + "cos_out": -0.02411760948598385, + "align_raw": -0.0719074010848999, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": 0.7254567927101394, + "R_pred_norm": -0.3453894660395481, + "R_pred_raw": -0.09835457603355818, + "layer": 10, + "head": 5, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0006203989323694259, + "spec_norm": 9.375325108295202, + "frob": 56.18359795961362, + "SR": 35.91256474485841, + "cos_in": -0.07572263479232788, + "cos_out": 0.012569521553814411, + "align_raw": 0.03747683763504028, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": 0.18095662419847636, + "R_pred_norm": -0.010428019077649994, + "R_pred_raw": -0.0033163447444124177, + "layer": 10, + "head": 6, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0010499250784050673, + "spec_norm": 11.000900231717544, + "frob": 46.6323301878643, + "SR": 17.968746540541265, + "cos_in": -0.5801035761833191, + "cos_out": 0.007113754749298096, + "align_raw": 0.02120944857597351, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": 0.05075696102565151, + "R_pred_norm": -0.08978234435489173, + "R_pred_raw": -0.024332862620035475, + "layer": 10, + "head": 7, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.011855785734951496, + "spec_norm": 15.539008982908092, + "frob": 50.32241507503763, + "SR": 10.487604850112564, + "cos_in": -0.6510568261146545, + "cos_out": -0.038883060216903687, + "align_raw": -0.11593148857355118, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": 0.19940012456381054, + "R_pred_norm": 8.784822053524566, + "R_pred_raw": 1.6855848741982975, + "layer": 10, + "head": 8, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0002217913629465329, + "spec_norm": 9.145337160130344, + "frob": 32.93676432705462, + "SR": 12.970670360230072, + "cos_in": -0.03380228951573372, + "cos_out": 0.0013780054869130254, + "align_raw": 0.0041085826233029366, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": -0.043084614213756416, + "R_pred_norm": -0.00017796778265602217, + "R_pred_raw": -5.802067092584975e-05, + "layer": 10, + "head": 9, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.001834927505115047, + "spec_norm": 39.854820765890416, + "frob": 74.28405022067321, + "SR": 3.4739969204351477, + "cos_in": -0.6553364396095276, + "cos_out": -0.006361962296068668, + "align_raw": -0.018968692049384117, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": -0.13751110478188952, + "R_pred_norm": 0.5743216048571407, + "R_pred_raw": 0.04296557107140795, + "layer": 10, + "head": 10, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0010455338560859673, + "spec_norm": 7.553501039928774, + "frob": 30.829615793274776, + "SR": 16.658643653607324, + "cos_in": -0.6332042813301086, + "cos_out": 0.02255856618285179, + "align_raw": 0.06725777685642242, + "norm_xA": 144.34605407714844, + "scale": 1883.6489791796112, + "R_obs": 0.09917426073640233, + "R_pred_norm": -0.21249156915874393, + "R_pred_raw": -0.08387343423276397, + "layer": 10, + "head": 11, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.012422895524650812, + "spec_norm": 61.65889700666178, + "frob": 86.9711978635025, + "SR": 1.9895708090537703, + "cos_in": -0.45635995268821716, + "cos_out": -0.06533711403608322, + "align_raw": -0.19480255246162415, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": 1.077084242826425, + "R_pred_norm": 63.83315485920713, + "R_pred_raw": 3.0866348351322954, + "layer": 11, + "head": 0, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.008369332586880773, + "spec_norm": 18.265131282089193, + "frob": 77.23712031080814, + "SR": 17.881607190408033, + "cos_in": 0.4048033654689789, + "cos_out": -0.0419570691883564, + "align_raw": -0.12509551644325256, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": 0.5089323385223748, + "R_pred_norm": -7.256432068745408, + "R_pred_raw": -1.1845051036551812, + "layer": 11, + "head": 1, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0006312266414170153, + "spec_norm": 18.919704826262237, + "frob": 106.23267321956556, + "SR": 31.527352839346545, + "cos_in": 0.1449865698814392, + "cos_out": -0.042750198394060135, + "align_raw": -0.12746146321296692, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": 0.284803985061183, + "R_pred_norm": -0.20688349178063534, + "R_pred_raw": -0.032602600797378374, + "layer": 11, + "head": 2, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0035179123806301504, + "spec_norm": 13.58022967182268, + "frob": 68.92652762860973, + "SR": 25.760754015986006, + "cos_in": -0.17983347177505493, + "cos_out": 0.0061498163267970085, + "align_raw": 0.018335778266191483, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": -0.5903600192851155, + "R_pred_norm": -0.14766754610940502, + "R_pred_raw": -0.032420159441590594, + "layer": 11, + "head": 3, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0158725930377841, + "spec_norm": 48.044989800413894, + "frob": 81.35082986381424, + "SR": 2.8670004694928535, + "cos_in": -0.6475040316581726, + "cos_out": -0.029556699097156525, + "align_raw": -0.08812528103590012, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": -0.3194322170224127, + "R_pred_norm": 40.790088251206974, + "R_pred_raw": 2.5313437435437467, + "layer": 11, + "head": 4, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.009660071576945484, + "spec_norm": 19.05751647844419, + "frob": 80.7581869882243, + "SR": 17.95727829022412, + "cos_in": -0.40534624457359314, + "cos_out": -0.048817336559295654, + "align_raw": -0.14554792642593384, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": 1.3862119940576791, + "R_pred_norm": 10.181391415377702, + "R_pred_raw": 1.592842267459281, + "layer": 11, + "head": 5, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.0021307473070919514, + "spec_norm": 18.683054315466933, + "frob": 97.40623997433379, + "SR": 27.181774531023397, + "cos_in": -0.520370364189148, + "cos_out": 0.045251231640577316, + "align_raw": 0.13491939008235931, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": 0.3427230476119693, + "R_pred_norm": -2.619889864194998, + "R_pred_raw": -0.41809899467957146, + "layer": 11, + "head": 6, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.021180948708206415, + "spec_norm": 16.053417680926426, + "frob": 76.0262341645961, + "SR": 22.42807228285055, + "cos_in": -0.011769319884479046, + "cos_out": -0.024886902421712875, + "align_raw": -0.07420039176940918, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": -0.26057355644356783, + "R_pred_norm": 0.2783526605429967, + "R_pred_raw": 0.051696748014873214, + "layer": 11, + "head": 7, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.00266912643564865, + "spec_norm": 334.85910049974757, + "frob": 338.5618019653133, + "SR": 1.022237249959398, + "cos_in": 0.44667163491249084, + "cos_out": -0.0009238416678272188, + "align_raw": -0.002754374872893095, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": 2.2317817717718826, + "R_pred_norm": -1.0308071548386784, + "R_pred_raw": -0.009177847129425733, + "layer": 11, + "head": 8, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.001548837186419405, + "spec_norm": 18.626947288356273, + "frob": 90.40253945893888, + "SR": 23.55471692205203, + "cos_in": -0.34268319606781006, + "cos_out": -0.015201035887002945, + "align_raw": -0.04532099515199661, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": 0.27238386808101867, + "R_pred_norm": 0.42002360362228613, + "R_pred_raw": 0.06722924967645703, + "layer": 11, + "head": 9, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.00947245059069246, + "spec_norm": 14.325500713871225, + "frob": 62.8168530829904, + "SR": 19.22793877092154, + "cos_in": -0.04972093924880028, + "cos_out": -0.010567376390099525, + "align_raw": -0.03150584548711777, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": 0.06895156125110266, + "R_pred_norm": 0.1992686188352342, + "R_pred_raw": 0.041471816891171985, + "layer": 11, + "head": 10, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + }, + { + "delta_attn": 0.003671654485515319, + "spec_norm": 62.309257767330415, + "frob": 95.02363501222999, + "SR": 2.3257237279019654, + "cos_in": 0.3457675874233246, + "cos_out": 0.09959856420755386, + "align_raw": 0.2969619929790497, + "norm_xA": 214.17323303222656, + "scale": 2794.8612405652, + "R_obs": 0.4668433248681506, + "R_pred_norm": 22.019730138911036, + "R_pred_raw": 1.0536762854001023, + "layer": 11, + "head": 11, + "word_A": "we", + "word_B": " have", + "gap": 0.07663106918334961 + } +] \ No newline at end of file diff --git a/data/mi4_svd_wqk/gpt2-large.json b/data/mi4_svd_wqk/gpt2-large.json new file mode 100644 index 0000000000000000000000000000000000000000..13fcaba76c87a82e8f72ee17a6d9aac40d5d2c90 --- /dev/null +++ b/data/mi4_svd_wqk/gpt2-large.json @@ -0,0 +1,5338 @@ +{ + "model": "gpt2-large", + "N": 36, + "L_crit": 33, + "n_heads": 20, + "d_head": 64, + "n_active": null, + "layer_stats": { + "0": { + "mean_stable_rank": 14.47738881363044, + "mean_spectral_entropy": 3.9223042011260985, + "mean_top_singular_frac": 0.05049941707402468, + "max_stable_rank_head": 10, + "per_head": [ + { + "head": 0, + "stable_rank": 27.073302980001117, + "spectral_entropy": 4.107427597045898, + "top_singular_frac": 0.025029879063367844, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 20.152601840256917, + "spectral_entropy": 4.073379039764404, + "top_singular_frac": 0.029939446598291397, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 24.668101517388216, + "spectral_entropy": 4.1038970947265625, + "top_singular_frac": 0.02630704641342163, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 30.783903266697767, + "spectral_entropy": 4.111706256866455, + "top_singular_frac": 0.02343071810901165, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 16.562480128888676, + "spectral_entropy": 4.098981857299805, + "top_singular_frac": 0.03238622471690178, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 13.047779067344951, + "spectral_entropy": 4.0615105628967285, + "top_singular_frac": 0.0377487987279892, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 8.10863356315776, + "spectral_entropy": 3.901918411254883, + "top_singular_frac": 0.05534869804978371, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.42288879556713, + "spectral_entropy": 3.9747707843780518, + "top_singular_frac": 0.05374915152788162, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 20.160956558920418, + "spectral_entropy": 4.100759506225586, + "top_singular_frac": 0.02924201264977455, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 5.081697052813239, + "spectral_entropy": 3.572178840637207, + "top_singular_frac": 0.08652791380882263, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 31.028134298234068, + "spectral_entropy": 4.110184669494629, + "top_singular_frac": 0.023332497105002403, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 13.903532050159784, + "spectral_entropy": 4.037125587463379, + "top_singular_frac": 0.037166398018598557, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 5.808000471103579, + "spectral_entropy": 3.7722527980804443, + "top_singular_frac": 0.07185129076242447, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 4.514321846742871, + "spectral_entropy": 3.5327627658843994, + "top_singular_frac": 0.09561572968959808, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 22.424978676269898, + "spectral_entropy": 4.077585220336914, + "top_singular_frac": 0.028063513338565826, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 4.046977514202265, + "spectral_entropy": 3.6399331092834473, + "top_singular_frac": 0.09479930251836777, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 16.155520471695148, + "spectral_entropy": 3.9939942359924316, + "top_singular_frac": 0.03563056141138077, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 8.990564352089827, + "spectral_entropy": 3.853794574737549, + "top_singular_frac": 0.05313115566968918, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 5.231036707571941, + "spectral_entropy": 3.593311071395874, + "top_singular_frac": 0.084129199385643, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 4.382365113503246, + "spectral_entropy": 3.728610038757324, + "top_singular_frac": 0.08655880391597748, + "rope_alignment": null + } + ] + }, + "1": { + "mean_stable_rank": 5.227110739349127, + "mean_spectral_entropy": 3.7005088329315186, + "mean_top_singular_frac": 0.09474837966263294, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 4.09346935697372, + "spectral_entropy": 3.652912139892578, + "top_singular_frac": 0.0928618535399437, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 1.9801598020165065, + "spectral_entropy": 3.107927083969116, + "top_singular_frac": 0.2111572027206421, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 8.047215938608414, + "spectral_entropy": 3.9462881088256836, + "top_singular_frac": 0.05239954590797424, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 3.8476006135262817, + "spectral_entropy": 3.458953380584717, + "top_singular_frac": 0.1194828525185585, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 5.876852442884399, + "spectral_entropy": 3.8746657371520996, + "top_singular_frac": 0.06514131277799606, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 1.7479832994096056, + "spectral_entropy": 3.2456703186035156, + "top_singular_frac": 0.20499834418296814, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 8.529730339901953, + "spectral_entropy": 3.9154717922210693, + "top_singular_frac": 0.052226580679416656, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 3.0960393727760374, + "spectral_entropy": 3.6768198013305664, + "top_singular_frac": 0.10695324838161469, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 13.316327567720114, + "spectral_entropy": 4.041165828704834, + "top_singular_frac": 0.037661533802747726, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 5.852818602199043, + "spectral_entropy": 3.8260092735290527, + "top_singular_frac": 0.06937340646982193, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 3.46997425354534, + "spectral_entropy": 3.8450284004211426, + "top_singular_frac": 0.08926847577095032, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 3.299915942182758, + "spectral_entropy": 3.4570822715759277, + "top_singular_frac": 0.11919191479682922, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 10.535207763485063, + "spectral_entropy": 4.086372375488281, + "top_singular_frac": 0.04107905551791191, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 2.941619590017695, + "spectral_entropy": 3.6468520164489746, + "top_singular_frac": 0.11332686990499496, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 3.8073119527338677, + "spectral_entropy": 3.2373290061950684, + "top_singular_frac": 0.1313154101371765, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 5.858589077736566, + "spectral_entropy": 3.7105469703674316, + "top_singular_frac": 0.07323987782001495, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 4.11993049617194, + "spectral_entropy": 3.721940755844116, + "top_singular_frac": 0.08890439569950104, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 4.049698946604754, + "spectral_entropy": 3.7418479919433594, + "top_singular_frac": 0.08785610646009445, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 4.700331896824483, + "spectral_entropy": 3.9437344074249268, + "top_singular_frac": 0.0697135478258133, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 5.371437531664013, + "spectral_entropy": 3.87355899810791, + "top_singular_frac": 0.06881605833768845, + "rope_alignment": null + } + ] + }, + "2": { + "mean_stable_rank": 5.187858631474805, + "mean_spectral_entropy": 3.7619433999061584, + "mean_top_singular_frac": 0.09059054702520371, + "max_stable_rank_head": 16, + "per_head": [ + { + "head": 0, + "stable_rank": 4.915867016380392, + "spectral_entropy": 3.939751625061035, + "top_singular_frac": 0.07035984843969345, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 4.733520985287207, + "spectral_entropy": 3.9040701389312744, + "top_singular_frac": 0.07191212475299835, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 4.945837819141163, + "spectral_entropy": 3.8432252407073975, + "top_singular_frac": 0.07326752692461014, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 4.049459345504472, + "spectral_entropy": 3.7698898315429688, + "top_singular_frac": 0.09169349819421768, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 5.76271856002288, + "spectral_entropy": 3.962369918823242, + "top_singular_frac": 0.06367543339729309, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 5.955122460521165, + "spectral_entropy": 3.6225242614746094, + "top_singular_frac": 0.07855156064033508, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 3.1893966614675358, + "spectral_entropy": 3.7420601844787598, + "top_singular_frac": 0.10106855630874634, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 3.538472888969755, + "spectral_entropy": 3.732611656188965, + "top_singular_frac": 0.09606136381626129, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 4.632714707119892, + "spectral_entropy": 4.074039459228516, + "top_singular_frac": 0.06432677805423737, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 3.441624377267803, + "spectral_entropy": 3.4311201572418213, + "top_singular_frac": 0.12894535064697266, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 4.4287858844808365, + "spectral_entropy": 3.869856834411621, + "top_singular_frac": 0.07702814042568207, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 2.7244680958507645, + "spectral_entropy": 3.3997206687927246, + "top_singular_frac": 0.14787432551383972, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 6.6835939311148795, + "spectral_entropy": 3.9192709922790527, + "top_singular_frac": 0.05952271819114685, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 8.073264060858302, + "spectral_entropy": 3.909825325012207, + "top_singular_frac": 0.05411691591143608, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 2.976036395497141, + "spectral_entropy": 3.5524137020111084, + "top_singular_frac": 0.12631377577781677, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 1.8879934590316148, + "spectral_entropy": 3.3137733936309814, + "top_singular_frac": 0.18803560733795166, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 12.9767660131114, + "spectral_entropy": 3.994189739227295, + "top_singular_frac": 0.03965790197253227, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 10.046664778386182, + "spectral_entropy": 4.011767864227295, + "top_singular_frac": 0.04455071687698364, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 6.749652658543387, + "spectral_entropy": 3.8982110023498535, + "top_singular_frac": 0.059687331318855286, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 2.0452125309393043, + "spectral_entropy": 3.3481760025024414, + "top_singular_frac": 0.1751614660024643, + "rope_alignment": null + } + ] + }, + "3": { + "mean_stable_rank": 6.143202398608626, + "mean_spectral_entropy": 3.7915947079658507, + "mean_top_singular_frac": 0.08552115820348263, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 9.317753340769082, + "spectral_entropy": 4.088056564331055, + "top_singular_frac": 0.0439998060464859, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 7.495566450151324, + "spectral_entropy": 3.9758639335632324, + "top_singular_frac": 0.05357925221323967, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 8.580285491938293, + "spectral_entropy": 3.8755533695220947, + "top_singular_frac": 0.0533108152449131, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 2.8841625937754607, + "spectral_entropy": 3.6222617626190186, + "top_singular_frac": 0.12461163848638535, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 6.2224393326782454, + "spectral_entropy": 3.866344928741455, + "top_singular_frac": 0.06399267166852951, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 4.80879869993282, + "spectral_entropy": 3.595632314682007, + "top_singular_frac": 0.08991450816392899, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 7.106087378637321, + "spectral_entropy": 3.9163055419921875, + "top_singular_frac": 0.05812773108482361, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 3.227794812813643, + "spectral_entropy": 3.3538856506347656, + "top_singular_frac": 0.13606005907058716, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 13.029573227030387, + "spectral_entropy": 4.008080959320068, + "top_singular_frac": 0.039071742445230484, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 3.0664460601504504, + "spectral_entropy": 3.6791253089904785, + "top_singular_frac": 0.10856350511312485, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 2.740777082143577, + "spectral_entropy": 3.413787603378296, + "top_singular_frac": 0.14421366155147552, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 5.596144001190499, + "spectral_entropy": 3.8533577919006348, + "top_singular_frac": 0.07124371826648712, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 6.814701724690988, + "spectral_entropy": 3.9119231700897217, + "top_singular_frac": 0.0584479458630085, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 8.505634799914366, + "spectral_entropy": 3.9758501052856445, + "top_singular_frac": 0.04979902133345604, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 4.787363764455539, + "spectral_entropy": 3.8998191356658936, + "top_singular_frac": 0.07462611049413681, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 3.4575716911400725, + "spectral_entropy": 3.808431386947632, + "top_singular_frac": 0.09294512867927551, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 6.917802001914277, + "spectral_entropy": 3.8285841941833496, + "top_singular_frac": 0.06351632624864578, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 12.559252014062135, + "spectral_entropy": 4.076223373413086, + "top_singular_frac": 0.03777052089571953, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 1.2839110911142453, + "spectral_entropy": 3.2680811882019043, + "top_singular_frac": 0.2669210433959961, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 4.461982413669753, + "spectral_entropy": 3.814725875854492, + "top_singular_frac": 0.07970795780420303, + "rope_alignment": null + } + ] + }, + "4": { + "mean_stable_rank": 6.860827759994744, + "mean_spectral_entropy": 3.8230819702148438, + "mean_top_singular_frac": 0.07772924210876227, + "max_stable_rank_head": 17, + "per_head": [ + { + "head": 0, + "stable_rank": 6.499298434857516, + "spectral_entropy": 3.888021469116211, + "top_singular_frac": 0.06134772673249245, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 10.590675623489833, + "spectral_entropy": 4.01363468170166, + "top_singular_frac": 0.043544188141822815, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 6.169049854059429, + "spectral_entropy": 3.935436487197876, + "top_singular_frac": 0.06164354085922241, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.1057724600626155, + "spectral_entropy": 3.954171895980835, + "top_singular_frac": 0.05737008899450302, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 3.4128131639360624, + "spectral_entropy": 3.6877150535583496, + "top_singular_frac": 0.10677725821733475, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 5.042822865659053, + "spectral_entropy": 3.7592251300811768, + "top_singular_frac": 0.08083024621009827, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 3.5546439755769463, + "spectral_entropy": 3.407794713973999, + "top_singular_frac": 0.1244351863861084, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 3.8877092271189975, + "spectral_entropy": 3.5942940711975098, + "top_singular_frac": 0.10218039155006409, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 6.5681815859290555, + "spectral_entropy": 4.0762786865234375, + "top_singular_frac": 0.05312706157565117, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 5.899837392007247, + "spectral_entropy": 3.8616318702697754, + "top_singular_frac": 0.06614062935113907, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 6.6726942600347146, + "spectral_entropy": 3.8697080612182617, + "top_singular_frac": 0.061169274151325226, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 4.44093582670122, + "spectral_entropy": 3.6868090629577637, + "top_singular_frac": 0.0875948816537857, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 8.475293318403274, + "spectral_entropy": 3.956287384033203, + "top_singular_frac": 0.05069469287991524, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 7.266352228586934, + "spectral_entropy": 3.9021267890930176, + "top_singular_frac": 0.05741416662931442, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 8.12789361651202, + "spectral_entropy": 3.9577484130859375, + "top_singular_frac": 0.05193571373820305, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 1.3530253032537862, + "spectral_entropy": 2.9989092350006104, + "top_singular_frac": 0.2877535820007324, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 4.822816926729051, + "spectral_entropy": 3.902087450027466, + "top_singular_frac": 0.07120486348867416, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 19.160723274856107, + "spectral_entropy": 4.054965972900391, + "top_singular_frac": 0.030930381268262863, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 10.674781641061198, + "spectral_entropy": 4.036491394042969, + "top_singular_frac": 0.04257417097687721, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 7.491234221059818, + "spectral_entropy": 3.918301582336426, + "top_singular_frac": 0.05591679736971855, + "rope_alignment": null + } + ] + }, + "5": { + "mean_stable_rank": 6.303258693247651, + "mean_spectral_entropy": 3.8952167510986326, + "mean_top_singular_frac": 0.0650800995528698, + "max_stable_rank_head": 15, + "per_head": [ + { + "head": 0, + "stable_rank": 5.221874255599396, + "spectral_entropy": 3.755617141723633, + "top_singular_frac": 0.07711325585842133, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 4.139418147792906, + "spectral_entropy": 3.89278507232666, + "top_singular_frac": 0.07815442234277725, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 5.294962766394812, + "spectral_entropy": 3.8942627906799316, + "top_singular_frac": 0.06843487173318863, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.819821459816087, + "spectral_entropy": 3.9703593254089355, + "top_singular_frac": 0.05277026444673538, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 6.015503861681668, + "spectral_entropy": 3.8767905235290527, + "top_singular_frac": 0.06447296589612961, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 7.853352469090587, + "spectral_entropy": 3.856365203857422, + "top_singular_frac": 0.05676712468266487, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 8.322173816188519, + "spectral_entropy": 4.049295425415039, + "top_singular_frac": 0.04806201532483101, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 4.923291209239786, + "spectral_entropy": 3.8998427391052246, + "top_singular_frac": 0.07119256258010864, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 3.614128127570085, + "spectral_entropy": 4.01593017578125, + "top_singular_frac": 0.07778654992580414, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 4.2028317777752795, + "spectral_entropy": 3.7366271018981934, + "top_singular_frac": 0.08992907404899597, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 3.928787498790977, + "spectral_entropy": 3.7912678718566895, + "top_singular_frac": 0.08690174669027328, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 5.590210914114667, + "spectral_entropy": 3.8377747535705566, + "top_singular_frac": 0.06847857683897018, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 8.241366843319424, + "spectral_entropy": 3.942328453063965, + "top_singular_frac": 0.05261282995343208, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 7.454584065985406, + "spectral_entropy": 3.916602611541748, + "top_singular_frac": 0.05662018433213234, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 7.759490884068344, + "spectral_entropy": 3.9588184356689453, + "top_singular_frac": 0.05398784577846527, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 8.497544048876074, + "spectral_entropy": 3.995009183883667, + "top_singular_frac": 0.04979147017002106, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 6.913685221648238, + "spectral_entropy": 3.8614296913146973, + "top_singular_frac": 0.05987345799803734, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 5.237458281068991, + "spectral_entropy": 3.716355323791504, + "top_singular_frac": 0.08101144433021545, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 7.245659215062007, + "spectral_entropy": 3.8691585063934326, + "top_singular_frac": 0.05848671868443489, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 7.789029000869762, + "spectral_entropy": 4.067714691162109, + "top_singular_frac": 0.0491546094417572, + "rope_alignment": null + } + ] + }, + "6": { + "mean_stable_rank": 6.623308729794722, + "mean_spectral_entropy": 3.877010536193848, + "mean_top_singular_frac": 0.06862668786197901, + "max_stable_rank_head": 17, + "per_head": [ + { + "head": 0, + "stable_rank": 2.4023487518850875, + "spectral_entropy": 3.720855712890625, + "top_singular_frac": 0.12279422581195831, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 8.189917844750587, + "spectral_entropy": 3.95416522026062, + "top_singular_frac": 0.05182240158319473, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 8.474328978108506, + "spectral_entropy": 3.93483829498291, + "top_singular_frac": 0.05198437720537186, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 3.0125514662595894, + "spectral_entropy": 3.960463523864746, + "top_singular_frac": 0.08998287469148636, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 3.8265110368765227, + "spectral_entropy": 3.761723041534424, + "top_singular_frac": 0.08946417272090912, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 10.569247420686363, + "spectral_entropy": 4.016249656677246, + "top_singular_frac": 0.04318103939294815, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 3.352775321523169, + "spectral_entropy": 3.8853487968444824, + "top_singular_frac": 0.08829963207244873, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 5.002777870299437, + "spectral_entropy": 3.8369321823120117, + "top_singular_frac": 0.07386202365159988, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 10.842089007108088, + "spectral_entropy": 3.8940954208374023, + "top_singular_frac": 0.046371422708034515, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.322406633963288, + "spectral_entropy": 3.9326868057250977, + "top_singular_frac": 0.04904511198401451, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 3.4045204764214083, + "spectral_entropy": 3.5929629802703857, + "top_singular_frac": 0.10933460295200348, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 8.16093272077928, + "spectral_entropy": 3.861173629760742, + "top_singular_frac": 0.056101951748132706, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 5.31239635720262, + "spectral_entropy": 3.7983999252319336, + "top_singular_frac": 0.07432994246482849, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 4.629480660966404, + "spectral_entropy": 3.928171396255493, + "top_singular_frac": 0.07146801799535751, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 3.852254877177176, + "spectral_entropy": 3.797769546508789, + "top_singular_frac": 0.08728862553834915, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.197170932592355, + "spectral_entropy": 3.9532394409179688, + "top_singular_frac": 0.04592090845108032, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 6.993266509038055, + "spectral_entropy": 3.917219877243042, + "top_singular_frac": 0.05826796963810921, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 11.092332807155465, + "spectral_entropy": 4.040813446044922, + "top_singular_frac": 0.04163988679647446, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 5.822930084270381, + "spectral_entropy": 3.845186233520508, + "top_singular_frac": 0.06675570458173752, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 8.005934838830651, + "spectral_entropy": 3.9079155921936035, + "top_singular_frac": 0.05461886525154114, + "rope_alignment": null + } + ] + }, + "7": { + "mean_stable_rank": 7.998942062348251, + "mean_spectral_entropy": 3.855079233646393, + "mean_top_singular_frac": 0.06525249239057303, + "max_stable_rank_head": 17, + "per_head": [ + { + "head": 0, + "stable_rank": 12.534066253077093, + "spectral_entropy": 3.96950364112854, + "top_singular_frac": 0.0412210188806057, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 7.5709319518993, + "spectral_entropy": 3.9083945751190186, + "top_singular_frac": 0.056015193462371826, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 6.484151087287451, + "spectral_entropy": 3.7559938430786133, + "top_singular_frac": 0.06775368005037308, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 6.000883062308359, + "spectral_entropy": 3.850632667541504, + "top_singular_frac": 0.06611055880784988, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 8.416792929415243, + "spectral_entropy": 3.8993582725524902, + "top_singular_frac": 0.05335071310400963, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 1.445612208069409, + "spectral_entropy": 3.8057291507720947, + "top_singular_frac": 0.17156246304512024, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 10.508000454998202, + "spectral_entropy": 3.9270505905151367, + "top_singular_frac": 0.04632792994379997, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 4.202227049119628, + "spectral_entropy": 3.6068646907806396, + "top_singular_frac": 0.09901028126478195, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 9.576634177432481, + "spectral_entropy": 3.90164852142334, + "top_singular_frac": 0.049536362290382385, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 7.786417146166008, + "spectral_entropy": 3.8614940643310547, + "top_singular_frac": 0.05700092390179634, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 5.751605067613988, + "spectral_entropy": 3.869478225708008, + "top_singular_frac": 0.0664185881614685, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 7.8442102484477045, + "spectral_entropy": 3.855329990386963, + "top_singular_frac": 0.05690690129995346, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 6.790395334748308, + "spectral_entropy": 3.792221784591675, + "top_singular_frac": 0.06463181227445602, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 3.75569916736994, + "spectral_entropy": 3.679988384246826, + "top_singular_frac": 0.09806904196739197, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 10.885845857275415, + "spectral_entropy": 3.9258742332458496, + "top_singular_frac": 0.0453009158372879, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 7.2927669698719315, + "spectral_entropy": 3.91520357131958, + "top_singular_frac": 0.057331763207912445, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 4.101307856103911, + "spectral_entropy": 3.836005210876465, + "top_singular_frac": 0.0812939926981926, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 16.620872371950583, + "spectral_entropy": 3.943570613861084, + "top_singular_frac": 0.03616181015968323, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 12.131623707105026, + "spectral_entropy": 3.8807296752929688, + "top_singular_frac": 0.044279273599386215, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 10.278798346705015, + "spectral_entropy": 3.916512966156006, + "top_singular_frac": 0.046766623854637146, + "rope_alignment": null + } + ] + }, + "8": { + "mean_stable_rank": 7.563764749229092, + "mean_spectral_entropy": 3.843186008930206, + "mean_top_singular_frac": 0.06806415393948555, + "max_stable_rank_head": 1, + "per_head": [ + { + "head": 0, + "stable_rank": 6.236866412481931, + "spectral_entropy": 3.8477773666381836, + "top_singular_frac": 0.0646134614944458, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 17.87846916567337, + "spectral_entropy": 3.995553970336914, + "top_singular_frac": 0.03344138711690903, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 3.204908919610329, + "spectral_entropy": 3.712780714035034, + "top_singular_frac": 0.10375599563121796, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 8.656006395916183, + "spectral_entropy": 3.9387264251708984, + "top_singular_frac": 0.05095232278108597, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 2.941803564343639, + "spectral_entropy": 3.4884095191955566, + "top_singular_frac": 0.1314111053943634, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 8.907680714704263, + "spectral_entropy": 3.912109851837158, + "top_singular_frac": 0.05101430043578148, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 4.23392481959037, + "spectral_entropy": 3.831228256225586, + "top_singular_frac": 0.08079308271408081, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 9.491976142602105, + "spectral_entropy": 3.941662311553955, + "top_singular_frac": 0.04849971830844879, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 4.244033177307451, + "spectral_entropy": 3.809041976928711, + "top_singular_frac": 0.08191491663455963, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.908283259270512, + "spectral_entropy": 3.9104533195495605, + "top_singular_frac": 0.04854785278439522, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.823446555392339, + "spectral_entropy": 3.9380595684051514, + "top_singular_frac": 0.04744361713528633, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 9.600296756254425, + "spectral_entropy": 3.840343713760376, + "top_singular_frac": 0.05237817391753197, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 10.062990200359843, + "spectral_entropy": 3.914719581604004, + "top_singular_frac": 0.04805716499686241, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 7.8648935142549865, + "spectral_entropy": 3.896172046661377, + "top_singular_frac": 0.055399298667907715, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 2.579182656097053, + "spectral_entropy": 3.9222159385681152, + "top_singular_frac": 0.10159166157245636, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.755999602013183, + "spectral_entropy": 3.9486029148101807, + "top_singular_frac": 0.045208435505628586, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 9.426820096801103, + "spectral_entropy": 3.877765655517578, + "top_singular_frac": 0.051349248737096786, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 8.57664546355775, + "spectral_entropy": 3.821546792984009, + "top_singular_frac": 0.05641086772084236, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 3.500787509269125, + "spectral_entropy": 3.552211046218872, + "top_singular_frac": 0.11212588846683502, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 3.380280059081893, + "spectral_entropy": 3.7643392086029053, + "top_singular_frac": 0.09637457877397537, + "rope_alignment": null + } + ] + }, + "9": { + "mean_stable_rank": 10.016432342067255, + "mean_spectral_entropy": 3.904832923412323, + "mean_top_singular_frac": 0.05171023067086935, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 3.1263497220632006, + "spectral_entropy": 3.7517499923706055, + "top_singular_frac": 0.10168516635894775, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 8.281778328454484, + "spectral_entropy": 3.855562210083008, + "top_singular_frac": 0.055279869586229324, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 15.629931804655094, + "spectral_entropy": 4.012524127960205, + "top_singular_frac": 0.035575881600379944, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 10.383450821518464, + "spectral_entropy": 3.8587286472320557, + "top_singular_frac": 0.049266524612903595, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 12.112434118203728, + "spectral_entropy": 3.8843469619750977, + "top_singular_frac": 0.04449232667684555, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 8.033362234546757, + "spectral_entropy": 3.8380448818206787, + "top_singular_frac": 0.05738263204693794, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 5.230629496321015, + "spectral_entropy": 3.856020212173462, + "top_singular_frac": 0.07181014120578766, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 10.959607012801278, + "spectral_entropy": 3.996293067932129, + "top_singular_frac": 0.04309500753879547, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 15.654058126068513, + "spectral_entropy": 3.975574016571045, + "top_singular_frac": 0.03645305335521698, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 13.873792982337656, + "spectral_entropy": 3.94844126701355, + "top_singular_frac": 0.03963686153292656, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 10.959971257924009, + "spectral_entropy": 4.023943901062012, + "top_singular_frac": 0.042612962424755096, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 9.71825701093422, + "spectral_entropy": 3.885340690612793, + "top_singular_frac": 0.049583930522203445, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 7.169491326907085, + "spectral_entropy": 3.8932862281799316, + "top_singular_frac": 0.05806707590818405, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 15.215834957328621, + "spectral_entropy": 4.041073799133301, + "top_singular_frac": 0.03517896309494972, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 8.236798223211887, + "spectral_entropy": 3.8620176315307617, + "top_singular_frac": 0.055775612592697144, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 8.755858694903148, + "spectral_entropy": 3.8943257331848145, + "top_singular_frac": 0.05243241414427757, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 8.035233448744306, + "spectral_entropy": 3.8876185417175293, + "top_singular_frac": 0.05497191473841667, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 9.057007204079833, + "spectral_entropy": 3.8441853523254395, + "top_singular_frac": 0.0532282292842865, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 9.675619049377683, + "spectral_entropy": 3.909491539001465, + "top_singular_frac": 0.04883763939142227, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 10.21918102096409, + "spectral_entropy": 3.878089666366577, + "top_singular_frac": 0.048838406801223755, + "rope_alignment": null + } + ] + }, + "10": { + "mean_stable_rank": 9.623633479897531, + "mean_spectral_entropy": 3.9016732573509216, + "mean_top_singular_frac": 0.05187304727733135, + "max_stable_rank_head": 5, + "per_head": [ + { + "head": 0, + "stable_rank": 11.014318130620419, + "spectral_entropy": 3.8875651359558105, + "top_singular_frac": 0.046592555940151215, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 9.440085093314927, + "spectral_entropy": 3.875471830368042, + "top_singular_frac": 0.050885170698165894, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 11.612396839609541, + "spectral_entropy": 3.9392895698547363, + "top_singular_frac": 0.043443623930215836, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 12.304221539239585, + "spectral_entropy": 3.9492411613464355, + "top_singular_frac": 0.04187526926398277, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 4.399242099623267, + "spectral_entropy": 3.835291862487793, + "top_singular_frac": 0.07897656410932541, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 15.431503776510738, + "spectral_entropy": 3.9751925468444824, + "top_singular_frac": 0.036656953394412994, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.042369396452141, + "spectral_entropy": 3.862217903137207, + "top_singular_frac": 0.04741809517145157, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.695593569748157, + "spectral_entropy": 3.8806679248809814, + "top_singular_frac": 0.05648360028862953, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 11.063273195149204, + "spectral_entropy": 3.87210750579834, + "top_singular_frac": 0.04697224125266075, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.898294014265318, + "spectral_entropy": 3.9454174041748047, + "top_singular_frac": 0.04714205488562584, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.74613128631379, + "spectral_entropy": 3.8979787826538086, + "top_singular_frac": 0.04908242076635361, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 10.306672341733615, + "spectral_entropy": 3.8958160877227783, + "top_singular_frac": 0.04809420555830002, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 8.202730063645179, + "spectral_entropy": 3.912698745727539, + "top_singular_frac": 0.0533427968621254, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 11.323918552435794, + "spectral_entropy": 3.9345524311065674, + "top_singular_frac": 0.04456634819507599, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 12.133035456234765, + "spectral_entropy": 3.931079626083374, + "top_singular_frac": 0.042819928377866745, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 6.060636634136419, + "spectral_entropy": 3.8572983741760254, + "top_singular_frac": 0.06480886042118073, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 12.555832481727078, + "spectral_entropy": 3.9751956462860107, + "top_singular_frac": 0.04071592912077904, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 7.335044343106092, + "spectral_entropy": 3.9467878341674805, + "top_singular_frac": 0.05491800978779793, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 6.000904301073335, + "spectral_entropy": 3.8686182498931885, + "top_singular_frac": 0.06546381860971451, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 4.906466483011273, + "spectral_entropy": 3.7909765243530273, + "top_singular_frac": 0.07720249891281128, + "rope_alignment": null + } + ] + }, + "11": { + "mean_stable_rank": 9.96236466710416, + "mean_spectral_entropy": 3.9123089790344237, + "mean_top_singular_frac": 0.049502033926546576, + "max_stable_rank_head": 9, + "per_head": [ + { + "head": 0, + "stable_rank": 7.988538926956659, + "spectral_entropy": 3.9061522483825684, + "top_singular_frac": 0.05407928675413132, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 11.707408381635931, + "spectral_entropy": 3.9953720569610596, + "top_singular_frac": 0.04153810814023018, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.06523124342427, + "spectral_entropy": 3.877570629119873, + "top_singular_frac": 0.04941362142562866, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 6.30500933722362, + "spectral_entropy": 3.887174367904663, + "top_singular_frac": 0.0622236505150795, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 7.820947283068501, + "spectral_entropy": 3.869391918182373, + "top_singular_frac": 0.05665026232600212, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 9.789825876070784, + "spectral_entropy": 3.9042723178863525, + "top_singular_frac": 0.04891651123762131, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 8.925813801418869, + "spectral_entropy": 3.9325857162475586, + "top_singular_frac": 0.05006195977330208, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.037018430678046, + "spectral_entropy": 3.893789768218994, + "top_singular_frac": 0.05854707956314087, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 9.472467842656219, + "spectral_entropy": 3.8736724853515625, + "top_singular_frac": 0.05073828622698784, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 17.141993209279246, + "spectral_entropy": 3.996490955352783, + "top_singular_frac": 0.03423306718468666, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 12.430595988113094, + "spectral_entropy": 3.9386773109436035, + "top_singular_frac": 0.042149145156145096, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 12.775968938512468, + "spectral_entropy": 3.9301953315734863, + "top_singular_frac": 0.041721466928720474, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 5.988649374361392, + "spectral_entropy": 3.862027883529663, + "top_singular_frac": 0.06542632728815079, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 11.29450048724088, + "spectral_entropy": 3.8685195446014404, + "top_singular_frac": 0.04658499360084534, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 8.024110506063185, + "spectral_entropy": 3.910806179046631, + "top_singular_frac": 0.05392960086464882, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.268398431499506, + "spectral_entropy": 3.9044203758239746, + "top_singular_frac": 0.047948308289051056, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 11.301580534353384, + "spectral_entropy": 3.904230833053589, + "top_singular_frac": 0.04508699104189873, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 10.576470254035094, + "spectral_entropy": 3.8896708488464355, + "top_singular_frac": 0.04745154082775116, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 8.481757315749693, + "spectral_entropy": 3.9920451641082764, + "top_singular_frac": 0.04928253963589668, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 11.851007179742394, + "spectral_entropy": 3.909113645553589, + "top_singular_frac": 0.0440579317510128, + "rope_alignment": null + } + ] + }, + "12": { + "mean_stable_rank": 10.99048786394631, + "mean_spectral_entropy": 3.927308166027069, + "mean_top_singular_frac": 0.04770421888679266, + "max_stable_rank_head": 11, + "per_head": [ + { + "head": 0, + "stable_rank": 10.413176116544923, + "spectral_entropy": 3.936511993408203, + "top_singular_frac": 0.04631989449262619, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 6.555491936686304, + "spectral_entropy": 3.885251045227051, + "top_singular_frac": 0.06301163882017136, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.224055285433677, + "spectral_entropy": 3.8894636631011963, + "top_singular_frac": 0.048172857612371445, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 9.649960556060414, + "spectral_entropy": 3.92185115814209, + "top_singular_frac": 0.04860704392194748, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 9.714103812237878, + "spectral_entropy": 3.8813607692718506, + "top_singular_frac": 0.04996224865317345, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 14.73665521897733, + "spectral_entropy": 3.996967315673828, + "top_singular_frac": 0.03696112707257271, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 9.533175062850113, + "spectral_entropy": 3.93430757522583, + "top_singular_frac": 0.04832997918128967, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 12.057819094200111, + "spectral_entropy": 3.9425384998321533, + "top_singular_frac": 0.042574647814035416, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 8.938712237389911, + "spectral_entropy": 3.899360179901123, + "top_singular_frac": 0.051655881106853485, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 4.688491959670954, + "spectral_entropy": 3.853351354598999, + "top_singular_frac": 0.07463641464710236, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 15.176870093424997, + "spectral_entropy": 4.005070209503174, + "top_singular_frac": 0.03626567870378494, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 20.49211258725831, + "spectral_entropy": 4.02895450592041, + "top_singular_frac": 0.030621979385614395, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 9.077546558867342, + "spectral_entropy": 3.8891079425811768, + "top_singular_frac": 0.0514066219329834, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 4.935102320547654, + "spectral_entropy": 4.001558303833008, + "top_singular_frac": 0.06501969695091248, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 13.680002916472484, + "spectral_entropy": 3.941251754760742, + "top_singular_frac": 0.040271542966365814, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.98430648362392, + "spectral_entropy": 3.8894219398498535, + "top_singular_frac": 0.04660186171531677, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 11.152948547933194, + "spectral_entropy": 3.896813154220581, + "top_singular_frac": 0.046008601784706116, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 14.190449558529568, + "spectral_entropy": 3.925652027130127, + "top_singular_frac": 0.03965252265334129, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 12.1011950319778, + "spectral_entropy": 3.9187471866607666, + "top_singular_frac": 0.04315703734755516, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 11.50758190023934, + "spectral_entropy": 3.9086227416992188, + "top_singular_frac": 0.04484710097312927, + "rope_alignment": null + } + ] + }, + "13": { + "mean_stable_rank": 10.368821163488269, + "mean_spectral_entropy": 3.9188125014305113, + "mean_top_singular_frac": 0.04846387524157762, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 10.693879948355905, + "spectral_entropy": 3.891408681869507, + "top_singular_frac": 0.047070350497961044, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 9.18413411599603, + "spectral_entropy": 3.905357837677002, + "top_singular_frac": 0.050284527242183685, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 8.024532906434674, + "spectral_entropy": 3.903116226196289, + "top_singular_frac": 0.054425086826086044, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 12.262101285938995, + "spectral_entropy": 3.9346213340759277, + "top_singular_frac": 0.042365893721580505, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 7.579013444369, + "spectral_entropy": 3.887483835220337, + "top_singular_frac": 0.05670824274420738, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 11.963040139914812, + "spectral_entropy": 4.0140228271484375, + "top_singular_frac": 0.04098603129386902, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.644011516558892, + "spectral_entropy": 3.925381898880005, + "top_singular_frac": 0.043913643807172775, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 9.901962935845972, + "spectral_entropy": 3.928334951400757, + "top_singular_frac": 0.047644589096307755, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 16.403551808060467, + "spectral_entropy": 4.030689239501953, + "top_singular_frac": 0.034320902079343796, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 11.281598006922048, + "spectral_entropy": 3.879423141479492, + "top_singular_frac": 0.04611928388476372, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 10.712539068501728, + "spectral_entropy": 3.9266233444213867, + "top_singular_frac": 0.04593675583600998, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 9.671235069783929, + "spectral_entropy": 3.9108352661132812, + "top_singular_frac": 0.04909082502126694, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 4.181359447836872, + "spectral_entropy": 3.8930721282958984, + "top_singular_frac": 0.07739821821451187, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 10.194171776792142, + "spectral_entropy": 3.934828281402588, + "top_singular_frac": 0.046728700399398804, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 10.349338644488714, + "spectral_entropy": 3.8933496475219727, + "top_singular_frac": 0.04782932624220848, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.06408531902934, + "spectral_entropy": 3.8848624229431152, + "top_singular_frac": 0.048810623586177826, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 13.513050735036746, + "spectral_entropy": 3.953021287918091, + "top_singular_frac": 0.04001447930932045, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 9.815708629179314, + "spectral_entropy": 3.899631977081299, + "top_singular_frac": 0.049189042299985886, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 12.475275354313323, + "spectral_entropy": 3.8835692405700684, + "top_singular_frac": 0.043831150978803635, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 7.461833116406462, + "spectral_entropy": 3.8966164588928223, + "top_singular_frac": 0.056609831750392914, + "rope_alignment": null + } + ] + }, + "14": { + "mean_stable_rank": 9.819520090021111, + "mean_spectral_entropy": 3.8844197750091554, + "mean_top_singular_frac": 0.05326705202460289, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 9.84970295745341, + "spectral_entropy": 3.8577613830566406, + "top_singular_frac": 0.05053066834807396, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 10.041550603226032, + "spectral_entropy": 3.8674697875976562, + "top_singular_frac": 0.04993206262588501, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 9.742569425607051, + "spectral_entropy": 3.8695309162139893, + "top_singular_frac": 0.050525858998298645, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 2.972860038831188, + "spectral_entropy": 3.718968391418457, + "top_singular_frac": 0.10702269524335861, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 12.76651948020782, + "spectral_entropy": 3.896327495574951, + "top_singular_frac": 0.0429338738322258, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 13.910861489660368, + "spectral_entropy": 3.905500888824463, + "top_singular_frac": 0.04060601070523262, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 12.892716877292452, + "spectral_entropy": 3.9238786697387695, + "top_singular_frac": 0.041870567947626114, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 12.417745077735066, + "spectral_entropy": 3.9968461990356445, + "top_singular_frac": 0.04053504019975662, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 15.891782291581016, + "spectral_entropy": 4.003137111663818, + "top_singular_frac": 0.03551662713289261, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 7.778561041606941, + "spectral_entropy": 3.9929890632629395, + "top_singular_frac": 0.05181494355201721, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 10.202413312809405, + "spectral_entropy": 3.87017822265625, + "top_singular_frac": 0.0492551214993, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 6.372029103442329, + "spectral_entropy": 3.8839869499206543, + "top_singular_frac": 0.06253635883331299, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 8.765113996262246, + "spectral_entropy": 3.8847591876983643, + "top_singular_frac": 0.0524221807718277, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 9.92244910578053, + "spectral_entropy": 3.9073867797851562, + "top_singular_frac": 0.048294175416231155, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 8.996807803367929, + "spectral_entropy": 3.8906869888305664, + "top_singular_frac": 0.051447611302137375, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 8.063519595506127, + "spectral_entropy": 3.8325960636138916, + "top_singular_frac": 0.05716198682785034, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 11.226642485532519, + "spectral_entropy": 3.8998289108276367, + "top_singular_frac": 0.04566764831542969, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 9.504018776838674, + "spectral_entropy": 3.8322181701660156, + "top_singular_frac": 0.052755869925022125, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 10.354237112660227, + "spectral_entropy": 4.000089645385742, + "top_singular_frac": 0.04434339702129364, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 4.718301225020906, + "spectral_entropy": 3.654254674911499, + "top_singular_frac": 0.09016834199428558, + "rope_alignment": null + } + ] + }, + "15": { + "mean_stable_rank": 10.586574614857517, + "mean_spectral_entropy": 3.9050893902778627, + "mean_top_singular_frac": 0.04938640464097262, + "max_stable_rank_head": 12, + "per_head": [ + { + "head": 0, + "stable_rank": 17.98243375091294, + "spectral_entropy": 4.048242568969727, + "top_singular_frac": 0.03239136561751366, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 6.7109030240962895, + "spectral_entropy": 3.8912947177886963, + "top_singular_frac": 0.06042284518480301, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 8.215899658353509, + "spectral_entropy": 3.839477777481079, + "top_singular_frac": 0.05652056261897087, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 10.122913287763858, + "spectral_entropy": 3.8733177185058594, + "top_singular_frac": 0.049282174557447433, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 10.47930659324186, + "spectral_entropy": 3.9312281608581543, + "top_singular_frac": 0.04624268040060997, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 14.411476318726619, + "spectral_entropy": 3.9492220878601074, + "top_singular_frac": 0.03873296082019806, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 9.054490971195948, + "spectral_entropy": 3.880134105682373, + "top_singular_frac": 0.05180679261684418, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.790671989434484, + "spectral_entropy": 3.8154454231262207, + "top_singular_frac": 0.05868272855877876, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 8.16829414213925, + "spectral_entropy": 3.8858795166015625, + "top_singular_frac": 0.05427549034357071, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 10.816044435124338, + "spectral_entropy": 3.8938112258911133, + "top_singular_frac": 0.046916522085666656, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.586316820534147, + "spectral_entropy": 3.86252498626709, + "top_singular_frac": 0.0510929673910141, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 13.883196319717168, + "spectral_entropy": 4.0005903244018555, + "top_singular_frac": 0.038005728274583817, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 21.2329881899446, + "spectral_entropy": 3.9996864795684814, + "top_singular_frac": 0.03056461736559868, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 9.91003813412983, + "spectral_entropy": 3.8883774280548096, + "top_singular_frac": 0.049332328140735626, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 6.987066277873532, + "spectral_entropy": 3.855409622192383, + "top_singular_frac": 0.060434356331825256, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.62562351298711, + "spectral_entropy": 3.8781485557556152, + "top_singular_frac": 0.04781486093997955, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 12.201023662683792, + "spectral_entropy": 3.93491268157959, + "top_singular_frac": 0.0424988754093647, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 5.644546780896274, + "spectral_entropy": 3.8388164043426514, + "top_singular_frac": 0.06852907687425613, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 11.249674320631891, + "spectral_entropy": 3.9093823432922363, + "top_singular_frac": 0.04554157331585884, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 6.658584106762929, + "spectral_entropy": 3.9258856773376465, + "top_singular_frac": 0.058639585971832275, + "rope_alignment": null + } + ] + }, + "16": { + "mean_stable_rank": 11.031872590984003, + "mean_spectral_entropy": 3.911960482597351, + "mean_top_singular_frac": 0.04896924737840891, + "max_stable_rank_head": 4, + "per_head": [ + { + "head": 0, + "stable_rank": 16.172894549731218, + "spectral_entropy": 3.9273436069488525, + "top_singular_frac": 0.03711046651005745, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 13.968785581368007, + "spectral_entropy": 3.9193482398986816, + "top_singular_frac": 0.04038727283477783, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 15.28832446832114, + "spectral_entropy": 4.007593631744385, + "top_singular_frac": 0.035743098706007004, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 8.441413545801078, + "spectral_entropy": 3.8961167335510254, + "top_singular_frac": 0.053487371653318405, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 17.852129252974557, + "spectral_entropy": 4.054260730743408, + "top_singular_frac": 0.03235583007335663, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 8.108814583641493, + "spectral_entropy": 3.8995606899261475, + "top_singular_frac": 0.05401857942342758, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 14.559780461255539, + "spectral_entropy": 4.020333290100098, + "top_singular_frac": 0.03668985515832901, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 10.352510305193771, + "spectral_entropy": 3.8938002586364746, + "top_singular_frac": 0.04781133309006691, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 14.205962762962454, + "spectral_entropy": 3.9027318954467773, + "top_singular_frac": 0.04031384363770485, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 16.43767927614807, + "spectral_entropy": 3.9130687713623047, + "top_singular_frac": 0.03723517060279846, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 4.909721008590225, + "spectral_entropy": 3.767744541168213, + "top_singular_frac": 0.07850250601768494, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 10.96933749951362, + "spectral_entropy": 3.9007232189178467, + "top_singular_frac": 0.04619854316115379, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 10.647757893678834, + "spectral_entropy": 3.8921055793762207, + "top_singular_frac": 0.04737604781985283, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 4.256368706860597, + "spectral_entropy": 3.8543076515197754, + "top_singular_frac": 0.07910756021738052, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 10.085664144450051, + "spectral_entropy": 3.854775905609131, + "top_singular_frac": 0.050061047077178955, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 5.860848065130661, + "spectral_entropy": 3.876098155975342, + "top_singular_frac": 0.06510519236326218, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 11.299974252199204, + "spectral_entropy": 3.930813789367676, + "top_singular_frac": 0.04461784288287163, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 7.992560911404347, + "spectral_entropy": 3.9050416946411133, + "top_singular_frac": 0.05425691977143288, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 11.290841075539257, + "spectral_entropy": 3.9160268306732178, + "top_singular_frac": 0.04496369510889053, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 7.936083474915934, + "spectral_entropy": 3.907414436340332, + "top_singular_frac": 0.05404277145862579, + "rope_alignment": null + } + ] + }, + "17": { + "mean_stable_rank": 10.076315839602705, + "mean_spectral_entropy": 3.8903931856155394, + "mean_top_singular_frac": 0.04978728443384171, + "max_stable_rank_head": 6, + "per_head": [ + { + "head": 0, + "stable_rank": 11.862272064739582, + "spectral_entropy": 3.8743457794189453, + "top_singular_frac": 0.0454397015273571, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 9.443612410824217, + "spectral_entropy": 3.858793258666992, + "top_singular_frac": 0.051462870091199875, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.260135957322055, + "spectral_entropy": 3.968803882598877, + "top_singular_frac": 0.04536072164773941, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.607418451593563, + "spectral_entropy": 3.852060556411743, + "top_singular_frac": 0.058193184435367584, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 10.769536222225952, + "spectral_entropy": 3.942734718322754, + "top_singular_frac": 0.045142196118831635, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 12.062898314503979, + "spectral_entropy": 3.892362356185913, + "top_singular_frac": 0.044191110879182816, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 16.80448252094831, + "spectral_entropy": 3.932602882385254, + "top_singular_frac": 0.0360640250146389, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 10.229364008447567, + "spectral_entropy": 3.8609814643859863, + "top_singular_frac": 0.04951993003487587, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 9.412608310632873, + "spectral_entropy": 3.9044277667999268, + "top_singular_frac": 0.04971284046769142, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 12.36440936573861, + "spectral_entropy": 3.907414436340332, + "top_singular_frac": 0.04318055510520935, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.720783988151997, + "spectral_entropy": 3.9051690101623535, + "top_singular_frac": 0.049076732248067856, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 5.934982522928553, + "spectral_entropy": 3.854156970977783, + "top_singular_frac": 0.06604067981243134, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 13.289553236954847, + "spectral_entropy": 3.9123153686523438, + "top_singular_frac": 0.041450951248407364, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 8.7697526655336, + "spectral_entropy": 3.8756048679351807, + "top_singular_frac": 0.05287094786763191, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 9.174957612419428, + "spectral_entropy": 3.859302282333374, + "top_singular_frac": 0.05250619724392891, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 9.580994605909677, + "spectral_entropy": 3.891777753829956, + "top_singular_frac": 0.049807678908109665, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 6.66500160899751, + "spectral_entropy": 3.864790439605713, + "top_singular_frac": 0.061592914164066315, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 9.577427814056232, + "spectral_entropy": 3.9176833629608154, + "top_singular_frac": 0.048757877200841904, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 9.286051706113295, + "spectral_entropy": 3.8586883544921875, + "top_singular_frac": 0.05201197788119316, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 8.710073404012283, + "spectral_entropy": 3.8738481998443604, + "top_singular_frac": 0.05336259678006172, + "rope_alignment": null + } + ] + }, + "18": { + "mean_stable_rank": 9.660302285233902, + "mean_spectral_entropy": 3.884758722782135, + "mean_top_singular_frac": 0.05338231734931469, + "max_stable_rank_head": 7, + "per_head": [ + { + "head": 0, + "stable_rank": 6.807668522441119, + "spectral_entropy": 3.824732780456543, + "top_singular_frac": 0.0630921944975853, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 10.366659399163392, + "spectral_entropy": 3.887763023376465, + "top_singular_frac": 0.04817769303917885, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 11.212864344493738, + "spectral_entropy": 3.9311845302581787, + "top_singular_frac": 0.044616665691137314, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 4.405580951564541, + "spectral_entropy": 3.750206470489502, + "top_singular_frac": 0.08567140996456146, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 9.722247998696007, + "spectral_entropy": 3.871123790740967, + "top_singular_frac": 0.050275444984436035, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 8.157694340616038, + "spectral_entropy": 3.844628095626831, + "top_singular_frac": 0.05637865886092186, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 15.91638756885612, + "spectral_entropy": 4.078499794006348, + "top_singular_frac": 0.03359164670109749, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 22.1254997004588, + "spectral_entropy": 3.9432616233825684, + "top_singular_frac": 0.03107498586177826, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 9.84123053470562, + "spectral_entropy": 3.918405055999756, + "top_singular_frac": 0.04818151518702507, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 12.839967764882772, + "spectral_entropy": 3.935772180557251, + "top_singular_frac": 0.0414205938577652, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 8.573945021524391, + "spectral_entropy": 3.8802924156188965, + "top_singular_frac": 0.05320047214627266, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 6.717232220687224, + "spectral_entropy": 3.8585140705108643, + "top_singular_frac": 0.06163380295038223, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 9.157756355280496, + "spectral_entropy": 3.920696973800659, + "top_singular_frac": 0.049823611974716187, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 6.125935884007751, + "spectral_entropy": 3.816857099533081, + "top_singular_frac": 0.06757398694753647, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 7.295053788579231, + "spectral_entropy": 3.873154640197754, + "top_singular_frac": 0.058848366141319275, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 7.761725971976482, + "spectral_entropy": 3.8801090717315674, + "top_singular_frac": 0.05658948794007301, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 10.497727803809521, + "spectral_entropy": 3.905177593231201, + "top_singular_frac": 0.04700789228081703, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 5.201097672069208, + "spectral_entropy": 3.81937837600708, + "top_singular_frac": 0.07268191874027252, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 9.468812238462398, + "spectral_entropy": 3.8496642112731934, + "top_singular_frac": 0.051905274391174316, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 11.010957622403232, + "spectral_entropy": 3.905752658843994, + "top_singular_frac": 0.045900724828243256, + "rope_alignment": null + } + ] + }, + "19": { + "mean_stable_rank": 9.731558322224434, + "mean_spectral_entropy": 3.909087121486664, + "mean_top_singular_frac": 0.05164922419935465, + "max_stable_rank_head": 11, + "per_head": [ + { + "head": 0, + "stable_rank": 4.244735630900502, + "spectral_entropy": 3.798529863357544, + "top_singular_frac": 0.08245085924863815, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 9.261014226944758, + "spectral_entropy": 3.8629183769226074, + "top_singular_frac": 0.0517202727496624, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.278458972286069, + "spectral_entropy": 3.876089334487915, + "top_singular_frac": 0.048692263662815094, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 6.305268881581217, + "spectral_entropy": 3.815680503845215, + "top_singular_frac": 0.06586632132530212, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 13.477815991765254, + "spectral_entropy": 3.9331536293029785, + "top_singular_frac": 0.040567751973867416, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 6.640919227465497, + "spectral_entropy": 3.902207851409912, + "top_singular_frac": 0.060098499059677124, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 12.770005017127795, + "spectral_entropy": 3.9668800830841064, + "top_singular_frac": 0.040702998638153076, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 10.002566968157227, + "spectral_entropy": 3.890735387802124, + "top_singular_frac": 0.04886233061552048, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 5.717791002185612, + "spectral_entropy": 3.882986068725586, + "top_singular_frac": 0.06556899845600128, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 16.764746915111164, + "spectral_entropy": 4.046285629272461, + "top_singular_frac": 0.03362293913960457, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 6.460810182803748, + "spectral_entropy": 3.9114623069763184, + "top_singular_frac": 0.06019311770796776, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 17.029794176188375, + "spectral_entropy": 3.959588050842285, + "top_singular_frac": 0.03529367595911026, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 8.64591030523045, + "spectral_entropy": 3.9059715270996094, + "top_singular_frac": 0.051867250353097916, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 11.097519589309698, + "spectral_entropy": 3.922146797180176, + "top_singular_frac": 0.04531730338931084, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 11.181720420724341, + "spectral_entropy": 3.9479763507843018, + "top_singular_frac": 0.04443875700235367, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 8.88886579803835, + "spectral_entropy": 3.927731990814209, + "top_singular_frac": 0.05060011148452759, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 11.881568930012344, + "spectral_entropy": 3.9198150634765625, + "top_singular_frac": 0.04364057257771492, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 7.832798886931318, + "spectral_entropy": 3.9462361335754395, + "top_singular_frac": 0.05297550931572914, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 7.20392203644414, + "spectral_entropy": 3.879760265350342, + "top_singular_frac": 0.058673590421676636, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 8.944933285280786, + "spectral_entropy": 3.885587215423584, + "top_singular_frac": 0.051831360906362534, + "rope_alignment": null + } + ] + }, + "20": { + "mean_stable_rank": 10.745459704783675, + "mean_spectral_entropy": 3.9080111145973206, + "mean_top_singular_frac": 0.05006580464541912, + "max_stable_rank_head": 1, + "per_head": [ + { + "head": 0, + "stable_rank": 10.648317981747063, + "spectral_entropy": 3.9472155570983887, + "top_singular_frac": 0.04518499597907066, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 19.36049765918957, + "spectral_entropy": 3.9638900756835938, + "top_singular_frac": 0.03288954868912697, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 9.313078867502403, + "spectral_entropy": 3.9197609424591064, + "top_singular_frac": 0.04938451945781708, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 5.131988734752243, + "spectral_entropy": 3.8234190940856934, + "top_singular_frac": 0.07402375340461731, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 14.295251491454161, + "spectral_entropy": 3.939406633377075, + "top_singular_frac": 0.03913645073771477, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 3.62914337714081, + "spectral_entropy": 3.7947187423706055, + "top_singular_frac": 0.09079530835151672, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.30117214362066, + "spectral_entropy": 3.92037034034729, + "top_singular_frac": 0.044776689261198044, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 13.923607285157928, + "spectral_entropy": 3.9103522300720215, + "top_singular_frac": 0.04039144888520241, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 4.149177622976755, + "spectral_entropy": 3.8304758071899414, + "top_singular_frac": 0.08136770129203796, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 8.934737912179529, + "spectral_entropy": 3.8294763565063477, + "top_singular_frac": 0.05436548963189125, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 12.344805600877091, + "spectral_entropy": 3.9231185913085938, + "top_singular_frac": 0.04252748563885689, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 13.392999962378804, + "spectral_entropy": 3.962531566619873, + "top_singular_frac": 0.03983396291732788, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 13.676152698519754, + "spectral_entropy": 4.03061580657959, + "top_singular_frac": 0.03767617791891098, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 13.760892155211856, + "spectral_entropy": 3.9532358646392822, + "top_singular_frac": 0.03947365656495094, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 10.710925897927225, + "spectral_entropy": 3.895317792892456, + "top_singular_frac": 0.046866875141859055, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 8.992216735757374, + "spectral_entropy": 3.9073452949523926, + "top_singular_frac": 0.050889644771814346, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 13.000763882657923, + "spectral_entropy": 3.90972900390625, + "top_singular_frac": 0.041894324123859406, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 10.230899806469465, + "spectral_entropy": 3.917433261871338, + "top_singular_frac": 0.047162458300590515, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 9.913416461127067, + "spectral_entropy": 3.8713979721069336, + "top_singular_frac": 0.04958145320415497, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 8.19914781902579, + "spectral_entropy": 3.9104113578796387, + "top_singular_frac": 0.05309414863586426, + "rope_alignment": null + } + ] + }, + "21": { + "mean_stable_rank": 9.195562352441039, + "mean_spectral_entropy": 3.8941832304000856, + "mean_top_singular_frac": 0.054077351838350295, + "max_stable_rank_head": 13, + "per_head": [ + { + "head": 0, + "stable_rank": 8.896135625958745, + "spectral_entropy": 3.8525476455688477, + "top_singular_frac": 0.053474511951208115, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 3.9772276720971695, + "spectral_entropy": 3.760714054107666, + "top_singular_frac": 0.0887078270316124, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 6.183121596345728, + "spectral_entropy": 3.871671199798584, + "top_singular_frac": 0.06375705450773239, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 8.862357595695366, + "spectral_entropy": 3.8557796478271484, + "top_singular_frac": 0.05333320051431656, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 10.908248912217775, + "spectral_entropy": 3.936199188232422, + "top_singular_frac": 0.044933244585990906, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 10.893904991557882, + "spectral_entropy": 3.9129843711853027, + "top_singular_frac": 0.045708540827035904, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 10.572957404359112, + "spectral_entropy": 3.9623281955718994, + "top_singular_frac": 0.0450625903904438, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.2386334607289315, + "spectral_entropy": 3.839446783065796, + "top_singular_frac": 0.060243669897317886, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 13.770549104806438, + "spectral_entropy": 3.9517807960510254, + "top_singular_frac": 0.039454299956560135, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 10.749164670808138, + "spectral_entropy": 3.8959038257598877, + "top_singular_frac": 0.046802643686532974, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 6.144682066686631, + "spectral_entropy": 3.800936222076416, + "top_singular_frac": 0.06801971793174744, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 6.968548823811891, + "spectral_entropy": 3.8704915046691895, + "top_singular_frac": 0.060375358909368515, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 9.605582089717384, + "spectral_entropy": 3.9339656829833984, + "top_singular_frac": 0.048171889036893845, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 15.348179208526506, + "spectral_entropy": 3.961827278137207, + "top_singular_frac": 0.03702738881111145, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 7.584758249233821, + "spectral_entropy": 3.9047720432281494, + "top_singular_frac": 0.05535992234945297, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 7.185759095351201, + "spectral_entropy": 3.8847479820251465, + "top_singular_frac": 0.05827545374631882, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 14.977857240113046, + "spectral_entropy": 3.955737829208374, + "top_singular_frac": 0.03779911622405052, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 8.013265374290388, + "spectral_entropy": 3.869964361190796, + "top_singular_frac": 0.05541050434112549, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 11.677666961803695, + "spectral_entropy": 4.018867492675781, + "top_singular_frac": 0.04115523770451546, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 4.352646904710917, + "spectral_entropy": 3.842998504638672, + "top_singular_frac": 0.07847486436367035, + "rope_alignment": null + } + ] + }, + "22": { + "mean_stable_rank": 8.565956270738315, + "mean_spectral_entropy": 3.8921114563941956, + "mean_top_singular_frac": 0.058423712477087976, + "max_stable_rank_head": 3, + "per_head": [ + { + "head": 0, + "stable_rank": 7.123999910745907, + "spectral_entropy": 3.8831958770751953, + "top_singular_frac": 0.058459971100091934, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 10.311804757316391, + "spectral_entropy": 3.9268383979797363, + "top_singular_frac": 0.046742863953113556, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 3.8322372137426943, + "spectral_entropy": 3.7160398960113525, + "top_singular_frac": 0.09359899163246155, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 17.351828787605324, + "spectral_entropy": 4.0667924880981445, + "top_singular_frac": 0.03248392418026924, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 4.430751941989925, + "spectral_entropy": 3.769672393798828, + "top_singular_frac": 0.08320416510105133, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 9.331516870783062, + "spectral_entropy": 3.889651298522949, + "top_singular_frac": 0.05066518858075142, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 7.00488924427728, + "spectral_entropy": 3.8759751319885254, + "top_singular_frac": 0.05932258814573288, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 11.851067617917531, + "spectral_entropy": 3.979124069213867, + "top_singular_frac": 0.04180751368403435, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 5.555992002222853, + "spectral_entropy": 3.8525960445404053, + "top_singular_frac": 0.068203404545784, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 4.353758275861171, + "spectral_entropy": 3.8381853103637695, + "top_singular_frac": 0.07946664094924927, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 16.057060096035173, + "spectral_entropy": 4.016728401184082, + "top_singular_frac": 0.03503202274441719, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 3.8029495476437907, + "spectral_entropy": 3.741903305053711, + "top_singular_frac": 0.09218396991491318, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 4.908970113345427, + "spectral_entropy": 3.9111242294311523, + "top_singular_frac": 0.06959462910890579, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 10.179265610360934, + "spectral_entropy": 3.94585919380188, + "top_singular_frac": 0.046268682926893234, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 8.84611119567777, + "spectral_entropy": 3.8936846256256104, + "top_singular_frac": 0.05173766240477562, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 5.809092928734018, + "spectral_entropy": 3.8015308380126953, + "top_singular_frac": 0.07016216218471527, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 9.64683905015718, + "spectral_entropy": 3.9658241271972656, + "top_singular_frac": 0.047211598604917526, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 12.505749426649187, + "spectral_entropy": 3.92252779006958, + "top_singular_frac": 0.04225262254476547, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 7.837302482148555, + "spectral_entropy": 3.9156782627105713, + "top_singular_frac": 0.05413554981350899, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 10.577938341552132, + "spectral_entropy": 3.92929744720459, + "top_singular_frac": 0.0459400974214077, + "rope_alignment": null + } + ] + }, + "23": { + "mean_stable_rank": 8.78153713784886, + "mean_spectral_entropy": 3.8883448243141174, + "mean_top_singular_frac": 0.05765502825379372, + "max_stable_rank_head": 17, + "per_head": [ + { + "head": 0, + "stable_rank": 12.130713881013044, + "spectral_entropy": 3.9408211708068848, + "top_singular_frac": 0.042299844324588776, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 8.40074657880017, + "spectral_entropy": 3.913334369659424, + "top_singular_frac": 0.05245158076286316, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 2.4000763062212034, + "spectral_entropy": 3.544379472732544, + "top_singular_frac": 0.14143094420433044, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 5.18990988419616, + "spectral_entropy": 3.9028983116149902, + "top_singular_frac": 0.06816423684358597, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 11.265297133132792, + "spectral_entropy": 3.9200961589813232, + "top_singular_frac": 0.044784627854824066, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 11.64774606163597, + "spectral_entropy": 3.902843952178955, + "top_singular_frac": 0.0446137972176075, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 6.820377760719501, + "spectral_entropy": 3.9001898765563965, + "top_singular_frac": 0.0590936616063118, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 12.29721084892373, + "spectral_entropy": 3.9378316402435303, + "top_singular_frac": 0.042148470878601074, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 10.916682872061374, + "spectral_entropy": 3.950198173522949, + "top_singular_frac": 0.044511448591947556, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.096215641723614, + "spectral_entropy": 3.895425796508789, + "top_singular_frac": 0.05114191770553589, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 10.801070182722587, + "spectral_entropy": 3.9096946716308594, + "top_singular_frac": 0.0463855005800724, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 7.6332760052206385, + "spectral_entropy": 3.9015679359436035, + "top_singular_frac": 0.05570248141884804, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 8.422308440351031, + "spectral_entropy": 3.927783489227295, + "top_singular_frac": 0.05174235999584198, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 6.269230160359491, + "spectral_entropy": 3.882473945617676, + "top_singular_frac": 0.06246108189225197, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 11.723421631433927, + "spectral_entropy": 3.972014904022217, + "top_singular_frac": 0.04235551506280899, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 8.106770107296576, + "spectral_entropy": 3.856358051300049, + "top_singular_frac": 0.055902428925037384, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 7.744168083523981, + "spectral_entropy": 3.8499557971954346, + "top_singular_frac": 0.057808149605989456, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 15.314513368929497, + "spectral_entropy": 4.021634101867676, + "top_singular_frac": 0.03562614321708679, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 5.273940678718904, + "spectral_entropy": 3.8295106887817383, + "top_singular_frac": 0.0717673972249031, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 4.1770671299930076, + "spectral_entropy": 3.8078839778900146, + "top_singular_frac": 0.08270897716283798, + "rope_alignment": null + } + ] + }, + "24": { + "mean_stable_rank": 8.924758186942997, + "mean_spectral_entropy": 3.916579043865204, + "mean_top_singular_frac": 0.05272684451192618, + "max_stable_rank_head": 10, + "per_head": [ + { + "head": 0, + "stable_rank": 6.987810434782609, + "spectral_entropy": 3.8996376991271973, + "top_singular_frac": 0.05833618342876434, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 8.934673768183416, + "spectral_entropy": 3.921475410461426, + "top_singular_frac": 0.05075562000274658, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 7.79888780755772, + "spectral_entropy": 3.903035879135132, + "top_singular_frac": 0.05525852367281914, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 5.84154052668233, + "spectral_entropy": 3.8552396297454834, + "top_singular_frac": 0.0670589804649353, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 5.538077870080738, + "spectral_entropy": 3.7895071506500244, + "top_singular_frac": 0.07249604910612106, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 12.07199903217964, + "spectral_entropy": 3.9616127014160156, + "top_singular_frac": 0.04176994785666466, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.700799898526252, + "spectral_entropy": 3.9986979961395264, + "top_singular_frac": 0.04151055961847305, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.102296339531909, + "spectral_entropy": 3.881484031677246, + "top_singular_frac": 0.05885491520166397, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 6.588832441819012, + "spectral_entropy": 3.8694982528686523, + "top_singular_frac": 0.06151561439037323, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 7.87093932607583, + "spectral_entropy": 3.8874807357788086, + "top_singular_frac": 0.05528820678591728, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 12.577646262153054, + "spectral_entropy": 4.001123905181885, + "top_singular_frac": 0.039974410086870193, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 11.045486464513981, + "spectral_entropy": 3.988399028778076, + "top_singular_frac": 0.04302849993109703, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 6.556465499463028, + "spectral_entropy": 3.8567752838134766, + "top_singular_frac": 0.06299569457769394, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 10.813183178727302, + "spectral_entropy": 3.938159465789795, + "top_singular_frac": 0.044904157519340515, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 6.1697595528556395, + "spectral_entropy": 3.853637218475342, + "top_singular_frac": 0.06464209407567978, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 7.2368205293582655, + "spectral_entropy": 3.8788232803344727, + "top_singular_frac": 0.058112870901823044, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 11.442455409755121, + "spectral_entropy": 3.9623231887817383, + "top_singular_frac": 0.04298083111643791, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 11.264844937580044, + "spectral_entropy": 3.9766080379486084, + "top_singular_frac": 0.04312204197049141, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 12.106596612401374, + "spectral_entropy": 3.992001533508301, + "top_singular_frac": 0.04105331003665924, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 8.846047846632654, + "spectral_entropy": 3.916060447692871, + "top_singular_frac": 0.0508783794939518, + "rope_alignment": null + } + ] + }, + "25": { + "mean_stable_rank": 10.284102530081402, + "mean_spectral_entropy": 3.9259265661239624, + "mean_top_singular_frac": 0.048293810337781906, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 10.978389955458548, + "spectral_entropy": 3.913174629211426, + "top_singular_frac": 0.045702069997787476, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 9.291962827347067, + "spectral_entropy": 3.906883478164673, + "top_singular_frac": 0.049930233508348465, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 13.270581419953412, + "spectral_entropy": 4.033641338348389, + "top_singular_frac": 0.038099147379398346, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 12.211896183583415, + "spectral_entropy": 3.9362363815307617, + "top_singular_frac": 0.04234829172492027, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 5.847846925243692, + "spectral_entropy": 3.8053464889526367, + "top_singular_frac": 0.06995710730552673, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 10.977581825432376, + "spectral_entropy": 3.9329302310943604, + "top_singular_frac": 0.044797249138355255, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 6.019663937070197, + "spectral_entropy": 3.866150379180908, + "top_singular_frac": 0.06511092185974121, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 8.593339058617047, + "spectral_entropy": 3.9628071784973145, + "top_singular_frac": 0.04998575523495674, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 13.899837841802885, + "spectral_entropy": 3.958300828933716, + "top_singular_frac": 0.03902691975235939, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 12.26329964180573, + "spectral_entropy": 3.9593167304992676, + "top_singular_frac": 0.04170007258653641, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 13.168515311633705, + "spectral_entropy": 3.944727897644043, + "top_singular_frac": 0.040629491209983826, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 12.239136906456187, + "spectral_entropy": 3.908468246459961, + "top_singular_frac": 0.043242570012807846, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 6.833403700656026, + "spectral_entropy": 3.8424696922302246, + "top_singular_frac": 0.0618693046271801, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 7.771055061574157, + "spectral_entropy": 3.8617653846740723, + "top_singular_frac": 0.05686796084046364, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 8.405132913834807, + "spectral_entropy": 4.049297332763672, + "top_singular_frac": 0.04793492332100868, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 11.300697746352434, + "spectral_entropy": 3.9454293251037598, + "top_singular_frac": 0.04392765834927559, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 12.216406896511076, + "spectral_entropy": 3.931633234024048, + "top_singular_frac": 0.042425647377967834, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 10.056259217811888, + "spectral_entropy": 3.8949692249298096, + "top_singular_frac": 0.048667196184396744, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 9.61990192539774, + "spectral_entropy": 3.9335477352142334, + "top_singular_frac": 0.04801074415445328, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 10.717141305085647, + "spectral_entropy": 3.9314355850219727, + "top_singular_frac": 0.04564294219017029, + "rope_alignment": null + } + ] + }, + "26": { + "mean_stable_rank": 8.251956010676817, + "mean_spectral_entropy": 3.8979544639587402, + "mean_top_singular_frac": 0.05865203980356455, + "max_stable_rank_head": 19, + "per_head": [ + { + "head": 0, + "stable_rank": 6.340681171938669, + "spectral_entropy": 3.862612724304199, + "top_singular_frac": 0.0629551112651825, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 10.973904755380389, + "spectral_entropy": 3.9143316745758057, + "top_singular_frac": 0.045495323836803436, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 5.872775619440057, + "spectral_entropy": 3.9102911949157715, + "top_singular_frac": 0.06339139491319656, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 11.136734920992707, + "spectral_entropy": 3.9445571899414062, + "top_singular_frac": 0.04428925737738609, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 10.36747393990527, + "spectral_entropy": 3.9841060638427734, + "top_singular_frac": 0.044672753661870956, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 7.214976709789012, + "spectral_entropy": 3.8709771633148193, + "top_singular_frac": 0.059263311326503754, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 13.302066912385984, + "spectral_entropy": 3.9707226753234863, + "top_singular_frac": 0.03951288387179375, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 5.874456158891925, + "spectral_entropy": 3.837634325027466, + "top_singular_frac": 0.06715887784957886, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 7.18272636752464, + "spectral_entropy": 3.9047436714172363, + "top_singular_frac": 0.057384707033634186, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 3.9704314868804667, + "spectral_entropy": 3.920685052871704, + "top_singular_frac": 0.07802458852529526, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 6.789137883951672, + "spectral_entropy": 3.887951135635376, + "top_singular_frac": 0.05989297479391098, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 4.616081192374535, + "spectral_entropy": 3.750762939453125, + "top_singular_frac": 0.08281499892473221, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 11.818270908452286, + "spectral_entropy": 3.9399733543395996, + "top_singular_frac": 0.04306649789214134, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 7.670587716575755, + "spectral_entropy": 3.914950370788574, + "top_singular_frac": 0.05468551069498062, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 10.245817367238386, + "spectral_entropy": 3.9386425018310547, + "top_singular_frac": 0.0464274138212204, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 2.398987150265987, + "spectral_entropy": 3.716191530227661, + "top_singular_frac": 0.12251926958560944, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 10.517055072741586, + "spectral_entropy": 3.944016695022583, + "top_singular_frac": 0.045533474534749985, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 8.671877475298732, + "spectral_entropy": 3.8970601558685303, + "top_singular_frac": 0.05228486284613609, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 6.075332948525129, + "spectral_entropy": 3.8603992462158203, + "top_singular_frac": 0.06554989516735077, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 13.999744454983137, + "spectral_entropy": 3.9884796142578125, + "top_singular_frac": 0.03811768814921379, + "rope_alignment": null + } + ] + }, + "27": { + "mean_stable_rank": 10.011453461836332, + "mean_spectral_entropy": 3.9209052205085753, + "mean_top_singular_frac": 0.05053543131798506, + "max_stable_rank_head": 14, + "per_head": [ + { + "head": 0, + "stable_rank": 11.586056766021569, + "spectral_entropy": 3.985710620880127, + "top_singular_frac": 0.042158156633377075, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 11.25171868231118, + "spectral_entropy": 3.940660238265991, + "top_singular_frac": 0.044101230800151825, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.093030234715721, + "spectral_entropy": 3.939253091812134, + "top_singular_frac": 0.046747948974370956, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.272551548754957, + "spectral_entropy": 3.7593164443969727, + "top_singular_frac": 0.06438503414392471, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 9.339578571009593, + "spectral_entropy": 3.9818875789642334, + "top_singular_frac": 0.0470244400203228, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 6.535128380729701, + "spectral_entropy": 3.86596941947937, + "top_singular_frac": 0.062433212995529175, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 10.74625602662694, + "spectral_entropy": 3.9878430366516113, + "top_singular_frac": 0.04366292804479599, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.0001688960279, + "spectral_entropy": 3.8702750205993652, + "top_singular_frac": 0.06001041829586029, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 9.568663317939635, + "spectral_entropy": 3.9035253524780273, + "top_singular_frac": 0.04956072196364403, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.447580908922912, + "spectral_entropy": 3.896693468093872, + "top_singular_frac": 0.05005290359258652, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 12.425778947166567, + "spectral_entropy": 3.9719269275665283, + "top_singular_frac": 0.040996041148900986, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 6.35030255354962, + "spectral_entropy": 3.952897548675537, + "top_singular_frac": 0.05901064723730087, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 4.745612175740401, + "spectral_entropy": 3.840214252471924, + "top_singular_frac": 0.07554614543914795, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 13.427277414966904, + "spectral_entropy": 3.974325180053711, + "top_singular_frac": 0.03937140852212906, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 18.86496161628333, + "spectral_entropy": 3.980642795562744, + "top_singular_frac": 0.032950811088085175, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 4.868968402010437, + "spectral_entropy": 3.81235408782959, + "top_singular_frac": 0.07633444666862488, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 11.210491970889091, + "spectral_entropy": 3.9247241020202637, + "top_singular_frac": 0.044730570167303085, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 10.207355466187076, + "spectral_entropy": 3.9320168495178223, + "top_singular_frac": 0.04668252915143967, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 9.376756639327077, + "spectral_entropy": 3.9478814601898193, + "top_singular_frac": 0.048280443996191025, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 15.91083071754604, + "spectral_entropy": 3.9499869346618652, + "top_singular_frac": 0.03666858747601509, + "rope_alignment": null + } + ] + }, + "28": { + "mean_stable_rank": 9.414813804942495, + "mean_spectral_entropy": 3.923787009716034, + "mean_top_singular_frac": 0.050791339762508866, + "max_stable_rank_head": 7, + "per_head": [ + { + "head": 0, + "stable_rank": 8.594082060654074, + "spectral_entropy": 3.8908560276031494, + "top_singular_frac": 0.0527539923787117, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 7.567713929032847, + "spectral_entropy": 3.93605899810791, + "top_singular_frac": 0.05420118197798729, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 13.571267625488405, + "spectral_entropy": 3.941612720489502, + "top_singular_frac": 0.040296848863363266, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.520570176135512, + "spectral_entropy": 3.9603207111358643, + "top_singular_frac": 0.053418416529893875, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 12.137547532025097, + "spectral_entropy": 3.919877052307129, + "top_singular_frac": 0.04311012476682663, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 8.931262905644958, + "spectral_entropy": 3.911029577255249, + "top_singular_frac": 0.05087606608867645, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 8.700936591017571, + "spectral_entropy": 3.9185309410095215, + "top_singular_frac": 0.05129581317305565, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 14.620254009086628, + "spectral_entropy": 4.001576900482178, + "top_singular_frac": 0.03698593005537987, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 7.064161216393641, + "spectral_entropy": 3.8690497875213623, + "top_singular_frac": 0.05946941301226616, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.117811970445645, + "spectral_entropy": 3.958434581756592, + "top_singular_frac": 0.048580486327409744, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 7.95475261912073, + "spectral_entropy": 3.9116525650024414, + "top_singular_frac": 0.053976912051439285, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 14.256253912502277, + "spectral_entropy": 3.990818500518799, + "top_singular_frac": 0.03776789829134941, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 12.763517315810578, + "spectral_entropy": 3.9248206615448, + "top_singular_frac": 0.04181240499019623, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 7.51437466313343, + "spectral_entropy": 3.887040138244629, + "top_singular_frac": 0.056817010045051575, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 5.218141972925348, + "spectral_entropy": 3.8760266304016113, + "top_singular_frac": 0.06917738169431686, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 7.3617296212381556, + "spectral_entropy": 3.8963522911071777, + "top_singular_frac": 0.057144083082675934, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 11.554279811300066, + "spectral_entropy": 3.9671053886413574, + "top_singular_frac": 0.04274754226207733, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 10.123500944399979, + "spectral_entropy": 3.912523031234741, + "top_singular_frac": 0.04779105260968208, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 7.1160466761249825, + "spectral_entropy": 3.918053150177002, + "top_singular_frac": 0.05684242025017738, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 6.608070546369992, + "spectral_entropy": 3.884000539779663, + "top_singular_frac": 0.060761816799640656, + "rope_alignment": null + } + ] + }, + "29": { + "mean_stable_rank": 11.874396360053556, + "mean_spectral_entropy": 3.9263789534568785, + "mean_top_singular_frac": 0.047723072953522204, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 9.1337895244999, + "spectral_entropy": 3.9297120571136475, + "top_singular_frac": 0.04960085079073906, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 12.341688394460862, + "spectral_entropy": 3.9406116008758545, + "top_singular_frac": 0.04217594489455223, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 8.902360173169543, + "spectral_entropy": 3.882664680480957, + "top_singular_frac": 0.05219176784157753, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 17.132029226186557, + "spectral_entropy": 3.964467763900757, + "top_singular_frac": 0.03501121327280998, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 2.3163921130718563, + "spectral_entropy": 3.6008214950561523, + "top_singular_frac": 0.14093495905399323, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 14.751969023896583, + "spectral_entropy": 3.942302703857422, + "top_singular_frac": 0.03829679638147354, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.310889126405016, + "spectral_entropy": 3.92031192779541, + "top_singular_frac": 0.04473983496427536, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 12.798759448722732, + "spectral_entropy": 3.9211575984954834, + "top_singular_frac": 0.0418807789683342, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 18.410205098754737, + "spectral_entropy": 4.05159330368042, + "top_singular_frac": 0.03175043687224388, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 10.384313053643591, + "spectral_entropy": 3.908507823944092, + "top_singular_frac": 0.04704735428094864, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 10.892338479498196, + "spectral_entropy": 3.932304859161377, + "top_singular_frac": 0.04504683241248131, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 10.899336989635941, + "spectral_entropy": 3.9383575916290283, + "top_singular_frac": 0.04482031986117363, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 11.933127634483231, + "spectral_entropy": 3.9274706840515137, + "top_singular_frac": 0.04325917363166809, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 12.097951686181633, + "spectral_entropy": 3.959549903869629, + "top_singular_frac": 0.041879843920469284, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 8.422945905905967, + "spectral_entropy": 3.928370952606201, + "top_singular_frac": 0.0519096702337265, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 11.727195760366005, + "spectral_entropy": 3.964536190032959, + "top_singular_frac": 0.04247775301337242, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 10.27252884817937, + "spectral_entropy": 3.9443976879119873, + "top_singular_frac": 0.04606068506836891, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 12.375134584659511, + "spectral_entropy": 3.925299882888794, + "top_singular_frac": 0.042512282729148865, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 15.637329044891205, + "spectral_entropy": 3.9581615924835205, + "top_singular_frac": 0.03689828887581825, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 15.74764308445867, + "spectral_entropy": 3.986978769302368, + "top_singular_frac": 0.035966672003269196, + "rope_alignment": null + } + ] + }, + "30": { + "mean_stable_rank": 9.664489671673811, + "mean_spectral_entropy": 3.9088935613632203, + "mean_top_singular_frac": 0.05089750625193119, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 6.37945017958013, + "spectral_entropy": 3.8892111778259277, + "top_singular_frac": 0.06168084219098091, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 6.901336565222535, + "spectral_entropy": 3.8487958908081055, + "top_singular_frac": 0.06152356415987015, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.384082931386699, + "spectral_entropy": 3.915253162384033, + "top_singular_frac": 0.046763841062784195, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 13.457174312771844, + "spectral_entropy": 3.9776763916015625, + "top_singular_frac": 0.0391644611954689, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 8.487817439380317, + "spectral_entropy": 3.9020211696624756, + "top_singular_frac": 0.05252862721681595, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 4.747956998440382, + "spectral_entropy": 3.761383056640625, + "top_singular_frac": 0.08117340505123138, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 6.061145091444822, + "spectral_entropy": 3.818714141845703, + "top_singular_frac": 0.06748330593109131, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 11.778311673294834, + "spectral_entropy": 3.9471206665039062, + "top_singular_frac": 0.04288141056895256, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 13.97216740665767, + "spectral_entropy": 3.9469680786132812, + "top_singular_frac": 0.03927607089281082, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.416812891235478, + "spectral_entropy": 3.909050226211548, + "top_singular_frac": 0.049643173813819885, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.443418259899246, + "spectral_entropy": 3.90201473236084, + "top_singular_frac": 0.04970033839344978, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 12.93984126625712, + "spectral_entropy": 3.972017288208008, + "top_singular_frac": 0.04023604094982147, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 12.518667454277033, + "spectral_entropy": 3.9496865272521973, + "top_singular_frac": 0.041470784693956375, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 9.326522663563352, + "spectral_entropy": 3.975356340408325, + "top_singular_frac": 0.04765934869647026, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 10.6843356087892, + "spectral_entropy": 3.909506320953369, + "top_singular_frac": 0.04645930975675583, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 9.136691907962733, + "spectral_entropy": 3.905895709991455, + "top_singular_frac": 0.05062120035290718, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 9.227569429879107, + "spectral_entropy": 3.9191207885742188, + "top_singular_frac": 0.0496404729783535, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 10.953763193611868, + "spectral_entropy": 3.9227535724639893, + "top_singular_frac": 0.045371633023023605, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 9.935191397948673, + "spectral_entropy": 3.9254918098449707, + "top_singular_frac": 0.04755144566297531, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 7.5375367618731905, + "spectral_entropy": 3.8798341751098633, + "top_singular_frac": 0.05712084844708443, + "rope_alignment": null + } + ] + }, + "31": { + "mean_stable_rank": 10.949332121596978, + "mean_spectral_entropy": 3.9177454590797423, + "mean_top_singular_frac": 0.046736940555274484, + "max_stable_rank_head": 0, + "per_head": [ + { + "head": 0, + "stable_rank": 17.64250392806874, + "spectral_entropy": 3.950343608856201, + "top_singular_frac": 0.03482874855399132, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 10.731417363676263, + "spectral_entropy": 3.896336555480957, + "top_singular_frac": 0.046711768954992294, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 11.112013692493175, + "spectral_entropy": 3.9282445907592773, + "top_singular_frac": 0.0448295921087265, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.76225319702908, + "spectral_entropy": 3.908994436264038, + "top_singular_frac": 0.05542384460568428, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 12.830086769585469, + "spectral_entropy": 3.9080910682678223, + "top_singular_frac": 0.04231805354356766, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 9.530530057219128, + "spectral_entropy": 3.897590398788452, + "top_singular_frac": 0.04993313178420067, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 10.17475998921971, + "spectral_entropy": 3.908863067626953, + "top_singular_frac": 0.04754381254315376, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 12.159389841331043, + "spectral_entropy": 3.9003396034240723, + "top_singular_frac": 0.04393230006098747, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 13.580155221082926, + "spectral_entropy": 3.969334840774536, + "top_singular_frac": 0.039445389062166214, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.121702952335813, + "spectral_entropy": 3.8565006256103516, + "top_singular_frac": 0.052574869245290756, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 10.702166907128198, + "spectral_entropy": 3.9198408126831055, + "top_singular_frac": 0.046041883528232574, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 11.972843822121627, + "spectral_entropy": 3.9448413848876953, + "top_singular_frac": 0.0426110215485096, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 11.607784524757827, + "spectral_entropy": 3.9036731719970703, + "top_singular_frac": 0.04457835853099823, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 6.469928001225232, + "spectral_entropy": 3.9107537269592285, + "top_singular_frac": 0.06044517830014229, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 10.572525793252561, + "spectral_entropy": 3.8829259872436523, + "top_singular_frac": 0.04764877259731293, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 8.871365515008435, + "spectral_entropy": 3.9429798126220703, + "top_singular_frac": 0.04970483481884003, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 7.24008057910196, + "spectral_entropy": 3.9029810428619385, + "top_singular_frac": 0.057040825486183167, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 11.199418775424888, + "spectral_entropy": 3.9168829917907715, + "top_singular_frac": 0.04518352076411247, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 15.715316705266325, + "spectral_entropy": 3.9916791915893555, + "top_singular_frac": 0.03588121011853218, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 9.990398796611132, + "spectral_entropy": 3.9137122631073, + "top_singular_frac": 0.04806169494986534, + "rope_alignment": null + } + ] + }, + "32": { + "mean_stable_rank": 9.526734958479397, + "mean_spectral_entropy": 3.907236695289612, + "mean_top_singular_frac": 0.05114905890077352, + "max_stable_rank_head": 9, + "per_head": [ + { + "head": 0, + "stable_rank": 8.443484488218587, + "spectral_entropy": 3.9577672481536865, + "top_singular_frac": 0.050639696419239044, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 7.44200735141854, + "spectral_entropy": 3.881819248199463, + "top_singular_frac": 0.057439692318439484, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 6.393621648134042, + "spectral_entropy": 3.872089147567749, + "top_singular_frac": 0.06286574900150299, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 9.908255680552605, + "spectral_entropy": 3.8879804611206055, + "top_singular_frac": 0.049165427684783936, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 11.614854020867133, + "spectral_entropy": 3.914738416671753, + "top_singular_frac": 0.04446624219417572, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 5.551308386516923, + "spectral_entropy": 3.858607292175293, + "top_singular_frac": 0.06850288808345795, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.561745780204218, + "spectral_entropy": 3.921926975250244, + "top_singular_frac": 0.0440882109105587, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 8.67159275422255, + "spectral_entropy": 3.9052414894104004, + "top_singular_frac": 0.05178577080368996, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 13.407732897519047, + "spectral_entropy": 3.947571277618408, + "top_singular_frac": 0.04023348167538643, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 16.118356758036064, + "spectral_entropy": 3.972221851348877, + "top_singular_frac": 0.03593802824616432, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.19662001755331, + "spectral_entropy": 3.8960461616516113, + "top_singular_frac": 0.0506134033203125, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 8.387078238527756, + "spectral_entropy": 3.8763394355773926, + "top_singular_frac": 0.05424566566944122, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 9.861784787647865, + "spectral_entropy": 3.880545139312744, + "top_singular_frac": 0.04951847344636917, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 9.453991325990085, + "spectral_entropy": 3.8925247192382812, + "top_singular_frac": 0.05024199187755585, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 7.804466786144057, + "spectral_entropy": 3.8753156661987305, + "top_singular_frac": 0.056446682661771774, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 6.666185824550543, + "spectral_entropy": 3.900425434112549, + "top_singular_frac": 0.05982512980699539, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 10.996259813291514, + "spectral_entropy": 3.943279266357422, + "top_singular_frac": 0.044559162110090256, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 14.0889933857774, + "spectral_entropy": 3.9545698165893555, + "top_singular_frac": 0.038952067494392395, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 6.742794351117458, + "spectral_entropy": 3.9050133228302, + "top_singular_frac": 0.060078397393226624, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 8.223564873298246, + "spectral_entropy": 3.9007115364074707, + "top_singular_frac": 0.053375016897916794, + "rope_alignment": null + } + ] + }, + "33": { + "mean_stable_rank": 10.49278153341553, + "mean_spectral_entropy": 3.9207733392715456, + "mean_top_singular_frac": 0.04801317229866982, + "max_stable_rank_head": 17, + "per_head": [ + { + "head": 0, + "stable_rank": 13.782445214129257, + "spectral_entropy": 3.9705119132995605, + "top_singular_frac": 0.03905897215008736, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 12.41639864532951, + "spectral_entropy": 3.9480206966400146, + "top_singular_frac": 0.041774969547986984, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.334339350905443, + "spectral_entropy": 3.8728299140930176, + "top_singular_frac": 0.04865128546953201, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 11.511002335008934, + "spectral_entropy": 3.925428867340088, + "top_singular_frac": 0.04391542449593544, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 12.393511833586803, + "spectral_entropy": 3.9526724815368652, + "top_singular_frac": 0.04170828312635422, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 9.946732279960578, + "spectral_entropy": 3.9090335369110107, + "top_singular_frac": 0.04830921068787575, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 12.454645067322392, + "spectral_entropy": 3.927367687225342, + "top_singular_frac": 0.04226906970143318, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.687069604117838, + "spectral_entropy": 3.909593343734741, + "top_singular_frac": 0.05510780215263367, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 6.063066589698336, + "spectral_entropy": 3.852694272994995, + "top_singular_frac": 0.06585849821567535, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 11.083574479212063, + "spectral_entropy": 3.9208083152770996, + "top_singular_frac": 0.04533335566520691, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.93226178509905, + "spectral_entropy": 3.9661190509796143, + "top_singular_frac": 0.04622630029916763, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 5.129166032443647, + "spectral_entropy": 3.8648247718811035, + "top_singular_frac": 0.0705772191286087, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 8.253042114678964, + "spectral_entropy": 3.8945939540863037, + "top_singular_frac": 0.05381796136498451, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 12.940793876155187, + "spectral_entropy": 3.9667811393737793, + "top_singular_frac": 0.040233246982097626, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 11.331966455574682, + "spectral_entropy": 3.912316083908081, + "top_singular_frac": 0.04510604962706566, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.385247132809042, + "spectral_entropy": 3.9399447441101074, + "top_singular_frac": 0.04602270945906639, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 8.029514929721461, + "spectral_entropy": 3.863520622253418, + "top_singular_frac": 0.05613942816853523, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 14.151419683232085, + "spectral_entropy": 3.9817466735839844, + "top_singular_frac": 0.03810621798038483, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 12.939137440480373, + "spectral_entropy": 3.93198561668396, + "top_singular_frac": 0.041288089007139206, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 9.09029581884493, + "spectral_entropy": 3.9046730995178223, + "top_singular_frac": 0.05075935274362564, + "rope_alignment": null + } + ] + }, + "34": { + "mean_stable_rank": 8.503772422969302, + "mean_spectral_entropy": 3.902398335933685, + "mean_top_singular_frac": 0.055183549784123896, + "max_stable_rank_head": 13, + "per_head": [ + { + "head": 0, + "stable_rank": 11.410968009483485, + "spectral_entropy": 3.9270706176757812, + "top_singular_frac": 0.04421568289399147, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 8.64560830688734, + "spectral_entropy": 3.927436590194702, + "top_singular_frac": 0.05107353627681732, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 3.224877155746541, + "spectral_entropy": 3.853490114212036, + "top_singular_frac": 0.09194264560937881, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 8.364113596871789, + "spectral_entropy": 3.851473331451416, + "top_singular_frac": 0.055422697216272354, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 5.279360226356364, + "spectral_entropy": 3.8025941848754883, + "top_singular_frac": 0.07351288199424744, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 10.431434503237023, + "spectral_entropy": 3.948821544647217, + "top_singular_frac": 0.04561746492981911, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 7.653429627554011, + "spectral_entropy": 3.8652541637420654, + "top_singular_frac": 0.057405225932598114, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 9.902128207548465, + "spectral_entropy": 3.9273061752319336, + "top_singular_frac": 0.047690290957689285, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 8.283299172638182, + "spectral_entropy": 3.91025972366333, + "top_singular_frac": 0.05303477495908737, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.572819109108986, + "spectral_entropy": 3.920898914337158, + "top_singular_frac": 0.048713285475969315, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 5.444039060140536, + "spectral_entropy": 3.846263885498047, + "top_singular_frac": 0.06960055232048035, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 6.195340728094368, + "spectral_entropy": 3.8768043518066406, + "top_singular_frac": 0.06369995325803757, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 10.153607387893002, + "spectral_entropy": 3.9150280952453613, + "top_singular_frac": 0.047615595161914825, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 13.200455402338873, + "spectral_entropy": 3.9685280323028564, + "top_singular_frac": 0.03983338177204132, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 5.405806153335019, + "spectral_entropy": 3.95222544670105, + "top_singular_frac": 0.06441790610551834, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.791511348732316, + "spectral_entropy": 3.9282681941986084, + "top_singular_frac": 0.04566797986626625, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 10.426465243881644, + "spectral_entropy": 3.9280614852905273, + "top_singular_frac": 0.046310555189847946, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 8.980261893562018, + "spectral_entropy": 3.905599355697632, + "top_singular_frac": 0.05083168298006058, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 8.028044776847638, + "spectral_entropy": 3.8998184204101562, + "top_singular_frac": 0.05440320819616318, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 8.681878549128413, + "spectral_entropy": 3.892764091491699, + "top_singular_frac": 0.05266169458627701, + "rope_alignment": null + } + ] + }, + "35": { + "mean_stable_rank": 9.76867953584213, + "mean_spectral_entropy": 3.9282705664634703, + "mean_top_singular_frac": 0.05174430785700679, + "max_stable_rank_head": 13, + "per_head": [ + { + "head": 0, + "stable_rank": 13.272036282999482, + "spectral_entropy": 3.955826997756958, + "top_singular_frac": 0.04015671834349632, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 14.056710633438723, + "spectral_entropy": 4.009782791137695, + "top_singular_frac": 0.03761765733361244, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 7.746787487211163, + "spectral_entropy": 3.9118618965148926, + "top_singular_frac": 0.05490308254957199, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 9.647291744921244, + "spectral_entropy": 3.958439826965332, + "top_singular_frac": 0.047155458480119705, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 8.43500229020986, + "spectral_entropy": 3.901590347290039, + "top_singular_frac": 0.05246047303080559, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 5.13773216400544, + "spectral_entropy": 3.9276015758514404, + "top_singular_frac": 0.06728136539459229, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 14.917637589860858, + "spectral_entropy": 3.962585687637329, + "top_singular_frac": 0.037668626755476, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.338969375229187, + "spectral_entropy": 3.9277262687683105, + "top_singular_frac": 0.05623270943760872, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 8.855977203769203, + "spectral_entropy": 3.934844493865967, + "top_singular_frac": 0.05031878873705864, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 7.712855172483305, + "spectral_entropy": 3.9565253257751465, + "top_singular_frac": 0.053139958530664444, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 6.5563027577259145, + "spectral_entropy": 3.911076307296753, + "top_singular_frac": 0.060668084770441055, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 8.278975897971453, + "spectral_entropy": 3.9601807594299316, + "top_singular_frac": 0.05117030814290047, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 11.7071940572619, + "spectral_entropy": 3.9302420616149902, + "top_singular_frac": 0.043472375720739365, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 20.8815258259648, + "spectral_entropy": 4.0356245040893555, + "top_singular_frac": 0.03003084845840931, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 11.260363970904962, + "spectral_entropy": 3.9293229579925537, + "top_singular_frac": 0.04447324573993683, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 8.935215425909687, + "spectral_entropy": 3.919271469116211, + "top_singular_frac": 0.05064086616039276, + "rope_alignment": null + }, + { + "head": 16, + "stable_rank": 6.873436391469495, + "spectral_entropy": 3.8681507110595703, + "top_singular_frac": 0.060432810336351395, + "rope_alignment": null + }, + { + "head": 17, + "stable_rank": 3.277901129550042, + "spectral_entropy": 3.6858363151550293, + "top_singular_frac": 0.10390820354223251, + "rope_alignment": null + }, + { + "head": 18, + "stable_rank": 11.444730938002525, + "spectral_entropy": 3.932711601257324, + "top_singular_frac": 0.04386632516980171, + "rope_alignment": null + }, + { + "head": 19, + "stable_rank": 9.0369443779534, + "spectral_entropy": 3.94620943069458, + "top_singular_frac": 0.049288250505924225, + "rope_alignment": null + } + ] + } + } +} \ No newline at end of file diff --git a/data/mi4_svd_wqk/gpt2-medium.json b/data/mi4_svd_wqk/gpt2-medium.json new file mode 100644 index 0000000000000000000000000000000000000000..7d89d469196adcf11b435622193b8d699d616397 --- /dev/null +++ b/data/mi4_svd_wqk/gpt2-medium.json @@ -0,0 +1,2890 @@ +{ + "model": "gpt2-medium", + "N": 24, + "L_crit": 23, + "n_heads": 16, + "d_head": 64, + "n_active": null, + "layer_stats": { + "0": { + "mean_stable_rank": 2.9225962594414194, + "mean_spectral_entropy": 2.658791534602642, + "mean_top_singular_frac": 0.23482979484833777, + "max_stable_rank_head": 9, + "per_head": [ + { + "head": 0, + "stable_rank": 1.1277572630658512, + "spectral_entropy": 1.919986605644226, + "top_singular_frac": 0.4980331063270569, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 1.5875495345816708, + "spectral_entropy": 2.8669400215148926, + "top_singular_frac": 0.2449435442686081, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 1.582419252802452, + "spectral_entropy": 2.3394203186035156, + "top_singular_frac": 0.321262925863266, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 2.183858451782895, + "spectral_entropy": 2.227269172668457, + "top_singular_frac": 0.2689570486545563, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 2.6702627364854883, + "spectral_entropy": 2.7013204097747803, + "top_singular_frac": 0.18947847187519073, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 5.568662035141684, + "spectral_entropy": 3.4720935821533203, + "top_singular_frac": 0.08387009054422379, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 1.7025307609199691, + "spectral_entropy": 2.49697208404541, + "top_singular_frac": 0.27930736541748047, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 2.067943266215615, + "spectral_entropy": 2.0742034912109375, + "top_singular_frac": 0.28930363059043884, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 3.9387462178916253, + "spectral_entropy": 2.8705203533172607, + "top_singular_frac": 0.14061734080314636, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 8.720145023075833, + "spectral_entropy": 3.9070799350738525, + "top_singular_frac": 0.051798779517412186, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 4.821602895554229, + "spectral_entropy": 3.5047755241394043, + "top_singular_frac": 0.09025514870882034, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 2.1197707046071366, + "spectral_entropy": 2.63649320602417, + "top_singular_frac": 0.2269442081451416, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 1.9096068140798326, + "spectral_entropy": 2.7075047492980957, + "top_singular_frac": 0.23261532187461853, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 1.4833241980084277, + "spectral_entropy": 1.7825744152069092, + "top_singular_frac": 0.40824052691459656, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 2.8739676892388357, + "spectral_entropy": 2.9358787536621094, + "top_singular_frac": 0.16315099596977234, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 2.403393307611164, + "spectral_entropy": 2.0976319313049316, + "top_singular_frac": 0.2684982120990753, + "rope_alignment": null + } + ] + }, + "1": { + "mean_stable_rank": 4.9024095102418555, + "mean_spectral_entropy": 3.382708489894867, + "mean_top_singular_frac": 0.14186068414710462, + "max_stable_rank_head": 13, + "per_head": [ + { + "head": 0, + "stable_rank": 1.3920849370277222, + "spectral_entropy": 2.691056251525879, + "top_singular_frac": 0.31157031655311584, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 5.324981012566959, + "spectral_entropy": 3.950626850128174, + "top_singular_frac": 0.06645786017179489, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 9.924742373846927, + "spectral_entropy": 3.917948007583618, + "top_singular_frac": 0.04788472503423691, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 1.8355618169090824, + "spectral_entropy": 2.853363513946533, + "top_singular_frac": 0.23963229358196259, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 5.137144437783214, + "spectral_entropy": 4.021548271179199, + "top_singular_frac": 0.0639442652463913, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 3.5155634515314405, + "spectral_entropy": 3.1352455615997314, + "top_singular_frac": 0.14805728197097778, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 1.7035116498674632, + "spectral_entropy": 2.46112060546875, + "top_singular_frac": 0.28760668635368347, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 2.85169975207308, + "spectral_entropy": 3.331693649291992, + "top_singular_frac": 0.1503928154706955, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 2.820261329016776, + "spectral_entropy": 2.9598236083984375, + "top_singular_frac": 0.1826058328151703, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 6.947320673587206, + "spectral_entropy": 3.8744258880615234, + "top_singular_frac": 0.06050169840455055, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 1.8290434012628953, + "spectral_entropy": 2.903482437133789, + "top_singular_frac": 0.23325644433498383, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 9.247741317562845, + "spectral_entropy": 3.911956787109375, + "top_singular_frac": 0.05010991171002388, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 6.4363702642963405, + "spectral_entropy": 3.8764586448669434, + "top_singular_frac": 0.062034301459789276, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 13.39925572893762, + "spectral_entropy": 4.014790058135986, + "top_singular_frac": 0.03821684792637825, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 3.408005282234726, + "spectral_entropy": 3.575620174407959, + "top_singular_frac": 0.11463659256696701, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 2.665264735365386, + "spectral_entropy": 2.6441755294799805, + "top_singular_frac": 0.21286307275295258, + "rope_alignment": null + } + ] + }, + "2": { + "mean_stable_rank": 5.455591463602486, + "mean_spectral_entropy": 3.694382071495056, + "mean_top_singular_frac": 0.10038876044563949, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 2.898839546780251, + "spectral_entropy": 3.2165462970733643, + "top_singular_frac": 0.15607565641403198, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 1.8589110707025702, + "spectral_entropy": 3.1446571350097656, + "top_singular_frac": 0.21328897774219513, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 15.804716908585423, + "spectral_entropy": 4.065735816955566, + "top_singular_frac": 0.033825453370809555, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 4.293957162698604, + "spectral_entropy": 3.916027069091797, + "top_singular_frac": 0.07618000358343124, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 3.301771779515427, + "spectral_entropy": 3.882404088973999, + "top_singular_frac": 0.09244092553853989, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 5.587517324630035, + "spectral_entropy": 3.987095355987549, + "top_singular_frac": 0.0622614324092865, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 3.3692634946249997, + "spectral_entropy": 3.7759785652160645, + "top_singular_frac": 0.10075870156288147, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 3.5986844312941955, + "spectral_entropy": 3.568714141845703, + "top_singular_frac": 0.10429654270410538, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 16.38460715682159, + "spectral_entropy": 3.9431071281433105, + "top_singular_frac": 0.03630499541759491, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 3.276219183493044, + "spectral_entropy": 3.7254819869995117, + "top_singular_frac": 0.10420777648687363, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 3.4188787747716622, + "spectral_entropy": 3.85146427154541, + "top_singular_frac": 0.09427092224359512, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 2.1567618126511494, + "spectral_entropy": 3.194840431213379, + "top_singular_frac": 0.18917310237884521, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 6.4100647347374275, + "spectral_entropy": 3.751023292541504, + "top_singular_frac": 0.06764810532331467, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 3.0665801494427636, + "spectral_entropy": 3.3128252029418945, + "top_singular_frac": 0.1432255059480667, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 5.014605205662112, + "spectral_entropy": 3.8177666664123535, + "top_singular_frac": 0.07488780468702316, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 6.848084681228531, + "spectral_entropy": 3.9564456939697266, + "top_singular_frac": 0.0573742613196373, + "rope_alignment": null + } + ] + }, + "3": { + "mean_stable_rank": 5.980605684377626, + "mean_spectral_entropy": 3.8482421040534973, + "mean_top_singular_frac": 0.07101162313483655, + "max_stable_rank_head": 9, + "per_head": [ + { + "head": 0, + "stable_rank": 9.521611663701483, + "spectral_entropy": 3.9311892986297607, + "top_singular_frac": 0.04828471317887306, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 4.149405654921819, + "spectral_entropy": 3.668898820877075, + "top_singular_frac": 0.0916539803147316, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 6.10291481321519, + "spectral_entropy": 3.975641965866089, + "top_singular_frac": 0.059586066752672195, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.51743172714354, + "spectral_entropy": 3.9534153938293457, + "top_singular_frac": 0.05477675423026085, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 3.123479652609278, + "spectral_entropy": 3.663595676422119, + "top_singular_frac": 0.10825872421264648, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 7.563059541277973, + "spectral_entropy": 3.8322556018829346, + "top_singular_frac": 0.06051516532897949, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 4.1962898000880555, + "spectral_entropy": 3.538555145263672, + "top_singular_frac": 0.09802956134080887, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 5.5542438166106285, + "spectral_entropy": 3.7501466274261475, + "top_singular_frac": 0.07264725863933563, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 6.860351122809268, + "spectral_entropy": 3.953690528869629, + "top_singular_frac": 0.05762989819049835, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 11.037516016583044, + "spectral_entropy": 3.9800961017608643, + "top_singular_frac": 0.04378056526184082, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 4.869257349958895, + "spectral_entropy": 3.886256456375122, + "top_singular_frac": 0.07320230454206467, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 5.000226755798342, + "spectral_entropy": 3.899383544921875, + "top_singular_frac": 0.07117357850074768, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 3.785969507535697, + "spectral_entropy": 3.7913432121276855, + "top_singular_frac": 0.09029049426317215, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 6.404525776187974, + "spectral_entropy": 3.993129253387451, + "top_singular_frac": 0.05867667868733406, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 4.030860097613093, + "spectral_entropy": 3.7939305305480957, + "top_singular_frac": 0.08599869161844254, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 5.972547653987748, + "spectral_entropy": 3.960345506668091, + "top_singular_frac": 0.061681535094976425, + "rope_alignment": null + } + ] + }, + "4": { + "mean_stable_rank": 8.415484687775676, + "mean_spectral_entropy": 3.879008322954178, + "mean_top_singular_frac": 0.05713009461760521, + "max_stable_rank_head": 1, + "per_head": [ + { + "head": 0, + "stable_rank": 6.719301832536649, + "spectral_entropy": 3.8491463661193848, + "top_singular_frac": 0.06161496043205261, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 12.814075735036582, + "spectral_entropy": 3.9383997917175293, + "top_singular_frac": 0.0413719117641449, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 9.873275147946792, + "spectral_entropy": 3.8913283348083496, + "top_singular_frac": 0.04919055104255676, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.288026173123521, + "spectral_entropy": 3.874429941177368, + "top_singular_frac": 0.058943163603544235, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 10.09385549252985, + "spectral_entropy": 3.8426880836486816, + "top_singular_frac": 0.05050531402230263, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 11.222951375564506, + "spectral_entropy": 3.9746835231781006, + "top_singular_frac": 0.043373748660087585, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 4.075083430680078, + "spectral_entropy": 4.006924629211426, + "top_singular_frac": 0.0727822408080101, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 12.648409785282697, + "spectral_entropy": 3.9279823303222656, + "top_singular_frac": 0.04201819747686386, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 6.10025891768999, + "spectral_entropy": 3.8327255249023438, + "top_singular_frac": 0.06635379046201706, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 6.300370904641613, + "spectral_entropy": 3.8659729957580566, + "top_singular_frac": 0.06307021528482437, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 12.06523248772627, + "spectral_entropy": 3.937746047973633, + "top_singular_frac": 0.04269536957144737, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 9.85022348911906, + "spectral_entropy": 3.959617853164673, + "top_singular_frac": 0.046922557055950165, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 8.60755237438121, + "spectral_entropy": 3.8637356758117676, + "top_singular_frac": 0.05418695881962776, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 5.744840777508602, + "spectral_entropy": 3.6180129051208496, + "top_singular_frac": 0.0821954682469368, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 6.210808907139459, + "spectral_entropy": 3.888390064239502, + "top_singular_frac": 0.0630202516913414, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 5.033488173503913, + "spectral_entropy": 3.792349100112915, + "top_singular_frac": 0.07583681493997574, + "rope_alignment": null + } + ] + }, + "5": { + "mean_stable_rank": 9.378876181054808, + "mean_spectral_entropy": 3.8874295949935913, + "mean_top_singular_frac": 0.053067698143422604, + "max_stable_rank_head": 12, + "per_head": [ + { + "head": 0, + "stable_rank": 6.053663627381116, + "spectral_entropy": 3.8324027061462402, + "top_singular_frac": 0.06571708619594574, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 10.510367899366786, + "spectral_entropy": 3.88277006149292, + "top_singular_frac": 0.047770753502845764, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 11.75429771310005, + "spectral_entropy": 3.904888391494751, + "top_singular_frac": 0.04440419375896454, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 9.04981100388851, + "spectral_entropy": 3.8943092823028564, + "top_singular_frac": 0.051592208445072174, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 11.855913447400845, + "spectral_entropy": 3.908933639526367, + "top_singular_frac": 0.04413725808262825, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 14.032294190632104, + "spectral_entropy": 3.9172210693359375, + "top_singular_frac": 0.04000283032655716, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 9.007732859191238, + "spectral_entropy": 3.8769803047180176, + "top_singular_frac": 0.05211101099848747, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 8.217740141433987, + "spectral_entropy": 3.938325881958008, + "top_singular_frac": 0.0522768497467041, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 10.505636979507154, + "spectral_entropy": 3.8737192153930664, + "top_singular_frac": 0.04821440950036049, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 7.477382187668174, + "spectral_entropy": 3.8729748725891113, + "top_singular_frac": 0.057762980461120605, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 3.999355059477678, + "spectral_entropy": 3.8243632316589355, + "top_singular_frac": 0.08335936814546585, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 7.0889384339479244, + "spectral_entropy": 3.779006242752075, + "top_singular_frac": 0.06388620287179947, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 14.496267835627298, + "spectral_entropy": 3.910615921020508, + "top_singular_frac": 0.03942788764834404, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 10.743986243632648, + "spectral_entropy": 3.9485530853271484, + "top_singular_frac": 0.04514147341251373, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 6.011592865497523, + "spectral_entropy": 3.8869242668151855, + "top_singular_frac": 0.06443072855472565, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 9.257038409123895, + "spectral_entropy": 3.946885347366333, + "top_singular_frac": 0.048847928643226624, + "rope_alignment": null + } + ] + }, + "6": { + "mean_stable_rank": 11.378339650781518, + "mean_spectral_entropy": 3.9177545309066772, + "mean_top_singular_frac": 0.046752868918702006, + "max_stable_rank_head": 1, + "per_head": [ + { + "head": 0, + "stable_rank": 10.91925720266857, + "spectral_entropy": 3.929295063018799, + "top_singular_frac": 0.045258402824401855, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 20.42690583855814, + "spectral_entropy": 3.9910058975219727, + "top_singular_frac": 0.03166278079152107, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 8.573179251827476, + "spectral_entropy": 3.8924148082733154, + "top_singular_frac": 0.05267172306776047, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 10.099641776228077, + "spectral_entropy": 3.8634257316589355, + "top_singular_frac": 0.04926847293972969, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 7.853097670527667, + "spectral_entropy": 3.938847541809082, + "top_singular_frac": 0.05383741110563278, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 12.275997993796652, + "spectral_entropy": 3.9626331329345703, + "top_singular_frac": 0.04174031317234039, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.024069873228102, + "spectral_entropy": 3.8970816135406494, + "top_singular_frac": 0.046291202306747437, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.222764466983216, + "spectral_entropy": 3.905667543411255, + "top_singular_frac": 0.05718996375799179, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 10.307694269512123, + "spectral_entropy": 3.9103622436523438, + "top_singular_frac": 0.0474528893828392, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 17.198527224098065, + "spectral_entropy": 4.013122081756592, + "top_singular_frac": 0.0336575023829937, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 17.348438009126887, + "spectral_entropy": 3.9820261001586914, + "top_singular_frac": 0.034285008907318115, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 13.213695606683675, + "spectral_entropy": 3.8411967754364014, + "top_singular_frac": 0.043384213000535965, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 10.485160282534324, + "spectral_entropy": 3.9289793968200684, + "top_singular_frac": 0.04629071056842804, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 6.332928550952078, + "spectral_entropy": 3.8267531394958496, + "top_singular_frac": 0.06455401331186295, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 9.860336316051683, + "spectral_entropy": 3.948626756668091, + "top_singular_frac": 0.047099579125642776, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 8.911740079727537, + "spectral_entropy": 3.8526346683502197, + "top_singular_frac": 0.05340171605348587, + "rope_alignment": null + } + ] + }, + "7": { + "mean_stable_rank": 10.798978317532429, + "mean_spectral_entropy": 3.928966209292412, + "mean_top_singular_frac": 0.04999721911735833, + "max_stable_rank_head": 7, + "per_head": [ + { + "head": 0, + "stable_rank": 15.371519646794356, + "spectral_entropy": 3.9354336261749268, + "top_singular_frac": 0.037782032042741776, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 8.707171770286125, + "spectral_entropy": 3.9482038021087646, + "top_singular_frac": 0.050306010991334915, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 14.934122595538637, + "spectral_entropy": 3.9520890712738037, + "top_singular_frac": 0.037941426038742065, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 2.751937824830247, + "spectral_entropy": 3.71392560005188, + "top_singular_frac": 0.11301938444375992, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 12.47718519019181, + "spectral_entropy": 4.0003743171691895, + "top_singular_frac": 0.03998444229364395, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 10.494791057625745, + "spectral_entropy": 3.9576263427734375, + "top_singular_frac": 0.04522550478577614, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 9.65188683540785, + "spectral_entropy": 3.920278549194336, + "top_singular_frac": 0.048666469752788544, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 16.18605415946331, + "spectral_entropy": 3.961440086364746, + "top_singular_frac": 0.03608083724975586, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 12.747071024078886, + "spectral_entropy": 3.9456512928009033, + "top_singular_frac": 0.04123283550143242, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 10.809224372680427, + "spectral_entropy": 3.958474636077881, + "top_singular_frac": 0.044983915984630585, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 13.535088033388666, + "spectral_entropy": 3.916266918182373, + "top_singular_frac": 0.04089204967021942, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 4.058394556946589, + "spectral_entropy": 4.036080837249756, + "top_singular_frac": 0.07109812647104263, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 7.855608634901147, + "spectral_entropy": 3.905369281768799, + "top_singular_frac": 0.05459035560488701, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 12.412170506799685, + "spectral_entropy": 3.9311888217926025, + "top_singular_frac": 0.042584002017974854, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 9.95968839223304, + "spectral_entropy": 3.902867555618286, + "top_singular_frac": 0.04842844232916832, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.83173847935233, + "spectral_entropy": 3.8781886100769043, + "top_singular_frac": 0.047139670699834824, + "rope_alignment": null + } + ] + }, + "8": { + "mean_stable_rank": 10.321756725432866, + "mean_spectral_entropy": 3.9090825021266937, + "mean_top_singular_frac": 0.04996182327158749, + "max_stable_rank_head": 9, + "per_head": [ + { + "head": 0, + "stable_rank": 10.851552103874285, + "spectral_entropy": 3.898832321166992, + "top_singular_frac": 0.046418845653533936, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 10.444589423115666, + "spectral_entropy": 3.8283498287200928, + "top_singular_frac": 0.049833789467811584, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.742511009487012, + "spectral_entropy": 3.9317538738250732, + "top_singular_frac": 0.045626651495695114, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 9.521291708079213, + "spectral_entropy": 3.9014787673950195, + "top_singular_frac": 0.04994295537471771, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 7.352181913944821, + "spectral_entropy": 3.833488941192627, + "top_singular_frac": 0.06011316180229187, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 5.57922297409432, + "spectral_entropy": 3.8283743858337402, + "top_singular_frac": 0.06931382417678833, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 9.538577273747853, + "spectral_entropy": 3.8644986152648926, + "top_singular_frac": 0.051062826067209244, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 11.446843662234155, + "spectral_entropy": 3.9647722244262695, + "top_singular_frac": 0.043123144656419754, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 4.920347085761691, + "spectral_entropy": 3.7954208850860596, + "top_singular_frac": 0.07571475207805634, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 17.967613495054348, + "spectral_entropy": 4.011009216308594, + "top_singular_frac": 0.0330759696662426, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 11.750503045752707, + "spectral_entropy": 3.9708895683288574, + "top_singular_frac": 0.042449139058589935, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 6.091942277906116, + "spectral_entropy": 3.9053544998168945, + "top_singular_frac": 0.06243253871798515, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 13.554681536876235, + "spectral_entropy": 3.977681875228882, + "top_singular_frac": 0.03933496028184891, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 10.4089302401008, + "spectral_entropy": 3.864793300628662, + "top_singular_frac": 0.04853680357336998, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 14.926930880787653, + "spectral_entropy": 3.972270965576172, + "top_singular_frac": 0.03753012791275978, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.05038897610898, + "spectral_entropy": 3.9963507652282715, + "top_singular_frac": 0.04487968236207962, + "rope_alignment": null + } + ] + }, + "9": { + "mean_stable_rank": 12.032609312979243, + "mean_spectral_entropy": 3.8911894410848618, + "mean_top_singular_frac": 0.046938772313296795, + "max_stable_rank_head": 13, + "per_head": [ + { + "head": 0, + "stable_rank": 9.096993560460632, + "spectral_entropy": 3.8826732635498047, + "top_singular_frac": 0.051543835550546646, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 6.2071842021647985, + "spectral_entropy": 3.8129005432128906, + "top_singular_frac": 0.06616814434528351, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 13.664547211872026, + "spectral_entropy": 3.914057970046997, + "top_singular_frac": 0.04090377315878868, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 11.581763041871607, + "spectral_entropy": 3.879481792449951, + "top_singular_frac": 0.04569359868764877, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 11.428591821277253, + "spectral_entropy": 3.8299560546875, + "top_singular_frac": 0.04710233584046364, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 12.805725990184806, + "spectral_entropy": 3.9212870597839355, + "top_singular_frac": 0.042097412049770355, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 13.857307813229424, + "spectral_entropy": 3.9488019943237305, + "top_singular_frac": 0.039467621594667435, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 11.44375031229018, + "spectral_entropy": 3.878298282623291, + "top_singular_frac": 0.04587266594171524, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 14.818066131280036, + "spectral_entropy": 3.9299819469451904, + "top_singular_frac": 0.03867410495877266, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 14.674310471552303, + "spectral_entropy": 3.89329195022583, + "top_singular_frac": 0.04011152312159538, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 12.700935946859364, + "spectral_entropy": 3.9160146713256836, + "top_singular_frac": 0.04252453148365021, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 13.758482547140035, + "spectral_entropy": 3.9223852157592773, + "top_singular_frac": 0.04037754610180855, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 4.62870787777643, + "spectral_entropy": 3.7158467769622803, + "top_singular_frac": 0.08413265645503998, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 19.259564004215072, + "spectral_entropy": 3.9961600303649902, + "top_singular_frac": 0.03224675729870796, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 14.602239099382455, + "spectral_entropy": 3.95357084274292, + "top_singular_frac": 0.038314200937747955, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 7.993578976111475, + "spectral_entropy": 3.8643226623535156, + "top_singular_frac": 0.05578964948654175, + "rope_alignment": null + } + ] + }, + "10": { + "mean_stable_rank": 11.24379739170783, + "mean_spectral_entropy": 3.898247092962265, + "mean_top_singular_frac": 0.047986534889787436, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 12.229604794210687, + "spectral_entropy": 3.8879823684692383, + "top_singular_frac": 0.04417360946536064, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 11.723910202706005, + "spectral_entropy": 3.902625799179077, + "top_singular_frac": 0.04457532614469528, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 14.196937934889009, + "spectral_entropy": 3.9030494689941406, + "top_singular_frac": 0.04031350463628769, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.791477102193885, + "spectral_entropy": 3.8817830085754395, + "top_singular_frac": 0.0560714527964592, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 16.06979681568082, + "spectral_entropy": 4.00886869430542, + "top_singular_frac": 0.03523566946387291, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 9.378315244155177, + "spectral_entropy": 3.874195098876953, + "top_singular_frac": 0.05117858201265335, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 7.755162068223608, + "spectral_entropy": 3.861933469772339, + "top_singular_frac": 0.05689863860607147, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 4.504326996155626, + "spectral_entropy": 3.8730955123901367, + "top_singular_frac": 0.07541300356388092, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 18.186682335377782, + "spectral_entropy": 3.91680645942688, + "top_singular_frac": 0.03500396013259888, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 12.118013498361757, + "spectral_entropy": 3.9218175411224365, + "top_singular_frac": 0.04315945506095886, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 15.946243488703274, + "spectral_entropy": 3.880009651184082, + "top_singular_frac": 0.03822626546025276, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 8.772613354245404, + "spectral_entropy": 3.8991639614105225, + "top_singular_frac": 0.05176249146461487, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 8.191525577461537, + "spectral_entropy": 3.869202136993408, + "top_singular_frac": 0.05495579540729523, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 14.005149652157973, + "spectral_entropy": 3.904898166656494, + "top_singular_frac": 0.040520526468753815, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 10.006775198299259, + "spectral_entropy": 3.9142632484436035, + "top_singular_frac": 0.047858670353889465, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 9.024224004503452, + "spectral_entropy": 3.8722589015960693, + "top_singular_frac": 0.05243760719895363, + "rope_alignment": null + } + ] + }, + "11": { + "mean_stable_rank": 9.840209924142986, + "mean_spectral_entropy": 3.893975630402565, + "mean_top_singular_frac": 0.05164591898210347, + "max_stable_rank_head": 1, + "per_head": [ + { + "head": 0, + "stable_rank": 6.342385467895022, + "spectral_entropy": 3.8571763038635254, + "top_singular_frac": 0.06351035833358765, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 16.700804663649922, + "spectral_entropy": 3.931269884109497, + "top_singular_frac": 0.03643350303173065, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 11.615617877607985, + "spectral_entropy": 3.909295082092285, + "top_singular_frac": 0.044752296060323715, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 12.692228180500994, + "spectral_entropy": 3.909212112426758, + "top_singular_frac": 0.042454540729522705, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 6.234611633911289, + "spectral_entropy": 3.830496311187744, + "top_singular_frac": 0.0652841180562973, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 11.344250960479199, + "spectral_entropy": 3.911663055419922, + "top_singular_frac": 0.04494611546397209, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 9.80126581877692, + "spectral_entropy": 3.8964743614196777, + "top_singular_frac": 0.049271855503320694, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 11.577154906748781, + "spectral_entropy": 3.909165859222412, + "top_singular_frac": 0.04478517547249794, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 6.765693665535953, + "spectral_entropy": 3.8430395126342773, + "top_singular_frac": 0.06244037672877312, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 7.6035529359138785, + "spectral_entropy": 3.902712345123291, + "top_singular_frac": 0.05575545132160187, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 10.639859471786496, + "spectral_entropy": 3.9272279739379883, + "top_singular_frac": 0.04597911611199379, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 4.610918710643501, + "spectral_entropy": 3.8235819339752197, + "top_singular_frac": 0.07737544178962708, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 11.500729379044822, + "spectral_entropy": 3.9144115447998047, + "top_singular_frac": 0.04463451728224754, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 13.883490171143917, + "spectral_entropy": 3.974809169769287, + "top_singular_frac": 0.038693565875291824, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 7.0772256061836005, + "spectral_entropy": 3.9082648754119873, + "top_singular_frac": 0.05741238594055176, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 9.053569336465488, + "spectral_entropy": 3.8548097610473633, + "top_singular_frac": 0.05260588601231575, + "rope_alignment": null + } + ] + }, + "12": { + "mean_stable_rank": 11.160350187922381, + "mean_spectral_entropy": 3.8978561460971832, + "mean_top_singular_frac": 0.04957420029677451, + "max_stable_rank_head": 5, + "per_head": [ + { + "head": 0, + "stable_rank": 6.2760251724741, + "spectral_entropy": 3.7479825019836426, + "top_singular_frac": 0.06967319548130035, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 15.176455376012456, + "spectral_entropy": 3.90494441986084, + "top_singular_frac": 0.0389116071164608, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.619926592467753, + "spectral_entropy": 3.9007339477539062, + "top_singular_frac": 0.046732962131500244, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 11.822913479828143, + "spectral_entropy": 3.9124703407287598, + "top_singular_frac": 0.04395153746008873, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 11.981695372917308, + "spectral_entropy": 3.9072370529174805, + "top_singular_frac": 0.044019754976034164, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 18.153703455919434, + "spectral_entropy": 3.994438648223877, + "top_singular_frac": 0.03318355977535248, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 14.462359338006321, + "spectral_entropy": 3.9372591972351074, + "top_singular_frac": 0.03894902020692825, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 11.840198705259583, + "spectral_entropy": 3.91739821434021, + "top_singular_frac": 0.04372633993625641, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 10.537365278361857, + "spectral_entropy": 3.886775016784668, + "top_singular_frac": 0.047934193164110184, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 16.925101752166235, + "spectral_entropy": 3.968191385269165, + "top_singular_frac": 0.035194024443626404, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 5.240083500064812, + "spectral_entropy": 3.8471388816833496, + "top_singular_frac": 0.07070644199848175, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 3.714267255716044, + "spectral_entropy": 3.8438615798950195, + "top_singular_frac": 0.08593138307332993, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 9.291784387164476, + "spectral_entropy": 3.917348861694336, + "top_singular_frac": 0.04955050349235535, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 14.965939585261207, + "spectral_entropy": 3.9211020469665527, + "top_singular_frac": 0.03868356719613075, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 7.7456595645142405, + "spectral_entropy": 3.8610587120056152, + "top_singular_frac": 0.05707202106714249, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 9.812124190624138, + "spectral_entropy": 3.8977575302124023, + "top_singular_frac": 0.04896709322929382, + "rope_alignment": null + } + ] + }, + "13": { + "mean_stable_rank": 9.408949902909429, + "mean_spectral_entropy": 3.890072152018547, + "mean_top_singular_frac": 0.05326095852069557, + "max_stable_rank_head": 14, + "per_head": [ + { + "head": 0, + "stable_rank": 4.562361919752102, + "spectral_entropy": 3.841554641723633, + "top_singular_frac": 0.07697653025388718, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 11.42884558108716, + "spectral_entropy": 3.893439531326294, + "top_singular_frac": 0.0454561784863472, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 8.477576620391359, + "spectral_entropy": 3.8789730072021484, + "top_singular_frac": 0.05375275760889053, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 8.651055415280181, + "spectral_entropy": 3.856569766998291, + "top_singular_frac": 0.05432949215173721, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 10.032229986623411, + "spectral_entropy": 3.9068243503570557, + "top_singular_frac": 0.04817019775509834, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 8.691571794347592, + "spectral_entropy": 3.8727831840515137, + "top_singular_frac": 0.05333493649959564, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 9.090339198558041, + "spectral_entropy": 3.920991897583008, + "top_singular_frac": 0.05009715259075165, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 6.77978278813511, + "spectral_entropy": 3.844189167022705, + "top_singular_frac": 0.06152261048555374, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 5.554317075563763, + "spectral_entropy": 3.8527188301086426, + "top_singular_frac": 0.0686623603105545, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 12.9872642206044, + "spectral_entropy": 3.9429774284362793, + "top_singular_frac": 0.04096686467528343, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.738247712424753, + "spectral_entropy": 3.891618251800537, + "top_singular_frac": 0.04945136606693268, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 6.968262214073669, + "spectral_entropy": 3.9023964405059814, + "top_singular_frac": 0.05830368027091026, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 11.434834850961426, + "spectral_entropy": 3.9250245094299316, + "top_singular_frac": 0.04432043805718422, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 12.288403948865188, + "spectral_entropy": 3.921027183532715, + "top_singular_frac": 0.043005600571632385, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 18.301729329636345, + "spectral_entropy": 3.9812045097351074, + "top_singular_frac": 0.03344140574336052, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 5.556375790246351, + "spectral_entropy": 3.80886173248291, + "top_singular_frac": 0.07038376480340958, + "rope_alignment": null + } + ] + }, + "14": { + "mean_stable_rank": 11.030662194622689, + "mean_spectral_entropy": 3.9064042568206787, + "mean_top_singular_frac": 0.04835387575440109, + "max_stable_rank_head": 2, + "per_head": [ + { + "head": 0, + "stable_rank": 11.237053570778972, + "spectral_entropy": 3.9184088706970215, + "top_singular_frac": 0.04490932077169418, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 11.664262450996844, + "spectral_entropy": 3.9088950157165527, + "top_singular_frac": 0.044372476637363434, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 18.49560001283929, + "spectral_entropy": 3.994922161102295, + "top_singular_frac": 0.03296199440956116, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 12.711826533063695, + "spectral_entropy": 3.953145980834961, + "top_singular_frac": 0.04111485928297043, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 7.715292602300186, + "spectral_entropy": 3.911489963531494, + "top_singular_frac": 0.05469409003853798, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 13.53739259018197, + "spectral_entropy": 3.9151089191436768, + "top_singular_frac": 0.04098822548985481, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 7.1708850879147175, + "spectral_entropy": 3.8298144340515137, + "top_singular_frac": 0.06068817526102066, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 10.225739465918213, + "spectral_entropy": 3.8749570846557617, + "top_singular_frac": 0.04874899238348007, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 10.73325754568246, + "spectral_entropy": 3.9296035766601562, + "top_singular_frac": 0.045703258365392685, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 16.131381587871637, + "spectral_entropy": 3.9380173683166504, + "top_singular_frac": 0.03676088899374008, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 6.32577244151272, + "spectral_entropy": 3.873540163040161, + "top_singular_frac": 0.06291181594133377, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 16.95071370052087, + "spectral_entropy": 3.9369001388549805, + "top_singular_frac": 0.03586166352033615, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 6.579263265283548, + "spectral_entropy": 3.861565113067627, + "top_singular_frac": 0.06201251223683357, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 5.911817134261249, + "spectral_entropy": 3.8663110733032227, + "top_singular_frac": 0.06566515564918518, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 12.765439731483056, + "spectral_entropy": 3.9124975204467773, + "top_singular_frac": 0.042269643396139145, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 8.33489739335359, + "spectral_entropy": 3.877290725708008, + "top_singular_frac": 0.05399893969297409, + "rope_alignment": null + } + ] + }, + "15": { + "mean_stable_rank": 9.205900644297735, + "mean_spectral_entropy": 3.887443870306015, + "mean_top_singular_frac": 0.05390725424513221, + "max_stable_rank_head": 7, + "per_head": [ + { + "head": 0, + "stable_rank": 4.638305738583745, + "spectral_entropy": 3.8032150268554688, + "top_singular_frac": 0.07900942116975784, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 11.841625309552347, + "spectral_entropy": 3.9611687660217285, + "top_singular_frac": 0.04241170361638069, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 7.292900044070587, + "spectral_entropy": 3.89278507232666, + "top_singular_frac": 0.057396844029426575, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 5.85725566280565, + "spectral_entropy": 3.871756076812744, + "top_singular_frac": 0.06548969447612762, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 8.100208029585557, + "spectral_entropy": 3.9222488403320312, + "top_singular_frac": 0.05280805379152298, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 8.965760336468911, + "spectral_entropy": 3.9017019271850586, + "top_singular_frac": 0.0510026179254055, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 4.43098628643357, + "spectral_entropy": 3.7730164527893066, + "top_singular_frac": 0.08243042230606079, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 13.979718396224436, + "spectral_entropy": 3.9390406608581543, + "top_singular_frac": 0.03949717804789543, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 6.972405843379573, + "spectral_entropy": 3.849254608154297, + "top_singular_frac": 0.06136099249124527, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 8.54637769458717, + "spectral_entropy": 3.8869118690490723, + "top_singular_frac": 0.05320078134536743, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.91236298370327, + "spectral_entropy": 3.874582052230835, + "top_singular_frac": 0.04982921481132507, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 10.929491863019775, + "spectral_entropy": 3.8760433197021484, + "top_singular_frac": 0.04715782031416893, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 13.753663476182563, + "spectral_entropy": 3.9466617107391357, + "top_singular_frac": 0.03976384177803993, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 9.545935954267115, + "spectral_entropy": 3.8977956771850586, + "top_singular_frac": 0.04967087134718895, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 12.131685229022999, + "spectral_entropy": 3.9059553146362305, + "top_singular_frac": 0.04365753382444382, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.395727460876483, + "spectral_entropy": 3.8969645500183105, + "top_singular_frac": 0.047829076647758484, + "rope_alignment": null + } + ] + }, + "16": { + "mean_stable_rank": 10.482317802361326, + "mean_spectral_entropy": 3.9081677347421646, + "mean_top_singular_frac": 0.048034887528046966, + "max_stable_rank_head": 15, + "per_head": [ + { + "head": 0, + "stable_rank": 12.05194757110463, + "spectral_entropy": 3.916944980621338, + "top_singular_frac": 0.043356139212846756, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 9.898847485126367, + "spectral_entropy": 3.903101682662964, + "top_singular_frac": 0.04850006103515625, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 6.090425103057972, + "spectral_entropy": 3.8901467323303223, + "top_singular_frac": 0.0633314773440361, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 12.652834000602743, + "spectral_entropy": 3.9224350452423096, + "top_singular_frac": 0.0420738123357296, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 9.60022111166128, + "spectral_entropy": 3.8945679664611816, + "top_singular_frac": 0.04974360764026642, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 9.679426282822044, + "spectral_entropy": 3.8765501976013184, + "top_singular_frac": 0.05031595751643181, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 9.080810432348501, + "spectral_entropy": 3.864133358001709, + "top_singular_frac": 0.05232590436935425, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 9.451917803420832, + "spectral_entropy": 3.8857922554016113, + "top_singular_frac": 0.05051621049642563, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 11.267588729204748, + "spectral_entropy": 3.9718823432922363, + "top_singular_frac": 0.043097030371427536, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 15.057607554357077, + "spectral_entropy": 3.9442434310913086, + "top_singular_frac": 0.03796698898077011, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 7.589089898348294, + "spectral_entropy": 3.897321939468384, + "top_singular_frac": 0.05601802468299866, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 8.763549026649512, + "spectral_entropy": 3.882262706756592, + "top_singular_frac": 0.05268863961100578, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 9.765107811883537, + "spectral_entropy": 3.920253276824951, + "top_singular_frac": 0.04795726388692856, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 10.192646986953614, + "spectral_entropy": 3.9196879863739014, + "top_singular_frac": 0.04697346314787865, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 10.535400701088404, + "spectral_entropy": 3.926387071609497, + "top_singular_frac": 0.04612763598561287, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 16.039664339151653, + "spectral_entropy": 3.9149727821350098, + "top_singular_frac": 0.03756598383188248, + "rope_alignment": null + } + ] + }, + "17": { + "mean_stable_rank": 12.205833247970766, + "mean_spectral_entropy": 3.9281263053417206, + "mean_top_singular_frac": 0.04424049984663725, + "max_stable_rank_head": 0, + "per_head": [ + { + "head": 0, + "stable_rank": 19.018313657783214, + "spectral_entropy": 3.9977376461029053, + "top_singular_frac": 0.03245183452963829, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 13.09941193870629, + "spectral_entropy": 3.9293463230133057, + "top_singular_frac": 0.041046373546123505, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 14.03250314794282, + "spectral_entropy": 3.9860520362854004, + "top_singular_frac": 0.03807118907570839, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 12.228159265005363, + "spectral_entropy": 3.931333541870117, + "top_singular_frac": 0.0426742248237133, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 13.918099256169159, + "spectral_entropy": 3.9289464950561523, + "top_singular_frac": 0.039997659623622894, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 10.081553036184228, + "spectral_entropy": 3.8753743171691895, + "top_singular_frac": 0.04921863600611687, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 15.569122287873174, + "spectral_entropy": 4.039301872253418, + "top_singular_frac": 0.03495242819190025, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 14.359154609706245, + "spectral_entropy": 3.9594743251800537, + "top_singular_frac": 0.038387298583984375, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 11.439276451977126, + "spectral_entropy": 3.913386344909668, + "top_singular_frac": 0.04464380815625191, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 13.613196902190689, + "spectral_entropy": 3.972308397293091, + "top_singular_frac": 0.0391821451485157, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 8.149136661487239, + "spectral_entropy": 3.9005351066589355, + "top_singular_frac": 0.053652405738830566, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 7.087189926286287, + "spectral_entropy": 3.8129091262817383, + "top_singular_frac": 0.06249465420842171, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 11.951086410712582, + "spectral_entropy": 3.9227042198181152, + "top_singular_frac": 0.04337692633271217, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 9.839877282566754, + "spectral_entropy": 3.8724706172943115, + "top_singular_frac": 0.05033175274729729, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 13.08959706502494, + "spectral_entropy": 3.9294593334198, + "top_singular_frac": 0.04114960506558418, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 7.817654067916136, + "spectral_entropy": 3.878681182861328, + "top_singular_frac": 0.05621705576777458, + "rope_alignment": null + } + ] + }, + "18": { + "mean_stable_rank": 11.21289866272713, + "mean_spectral_entropy": 3.922125995159149, + "mean_top_singular_frac": 0.048331615049391985, + "max_stable_rank_head": 10, + "per_head": [ + { + "head": 0, + "stable_rank": 10.001412854279673, + "spectral_entropy": 3.9107449054718018, + "top_singular_frac": 0.048159580677747726, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 12.793156625977378, + "spectral_entropy": 3.9466304779052734, + "top_singular_frac": 0.041131068021059036, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 7.198577593994815, + "spectral_entropy": 3.836786985397339, + "top_singular_frac": 0.060781028121709824, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.847197381742432, + "spectral_entropy": 3.887880325317383, + "top_singular_frac": 0.055616240948438644, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 10.678257925552556, + "spectral_entropy": 3.96254825592041, + "top_singular_frac": 0.04475090652704239, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 17.598206576635903, + "spectral_entropy": 3.9569478034973145, + "top_singular_frac": 0.03473205491900444, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 7.403077837859752, + "spectral_entropy": 3.8073110580444336, + "top_singular_frac": 0.06153810769319534, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 11.307650519615116, + "spectral_entropy": 3.934523105621338, + "top_singular_frac": 0.044140081852674484, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 12.295760588308731, + "spectral_entropy": 3.9780876636505127, + "top_singular_frac": 0.040952417999506, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 11.007250869419053, + "spectral_entropy": 3.888148784637451, + "top_singular_frac": 0.046258579939603806, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 18.01295159626252, + "spectral_entropy": 3.9984583854675293, + "top_singular_frac": 0.03339298442006111, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 11.585987831176267, + "spectral_entropy": 3.9772515296936035, + "top_singular_frac": 0.04246210679411888, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 13.932612233224068, + "spectral_entropy": 3.974299669265747, + "top_singular_frac": 0.0386129692196846, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 16.698174281704464, + "spectral_entropy": 3.964968204498291, + "top_singular_frac": 0.03541388362646103, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 7.486434916833631, + "spectral_entropy": 3.9166221618652344, + "top_singular_frac": 0.05527253448963165, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 3.559668971047709, + "spectral_entropy": 3.8128066062927246, + "top_singular_frac": 0.0900912955403328, + "rope_alignment": null + } + ] + }, + "19": { + "mean_stable_rank": 12.186011496469789, + "mean_spectral_entropy": 3.9357545971870422, + "mean_top_singular_frac": 0.044136419193819165, + "max_stable_rank_head": 13, + "per_head": [ + { + "head": 0, + "stable_rank": 8.389907490088223, + "spectral_entropy": 3.8793187141418457, + "top_singular_frac": 0.05386321246623993, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 8.567988670488111, + "spectral_entropy": 3.922325611114502, + "top_singular_frac": 0.05161792039871216, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.450213551351576, + "spectral_entropy": 3.99006986618042, + "top_singular_frac": 0.04430549964308739, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 16.162549531042018, + "spectral_entropy": 3.980146884918213, + "top_singular_frac": 0.035625241696834564, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 8.187830264880434, + "spectral_entropy": 3.8848037719726562, + "top_singular_frac": 0.054384902119636536, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 12.487050715274762, + "spectral_entropy": 3.9331281185150146, + "top_singular_frac": 0.04200746491551399, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 6.139382129266922, + "spectral_entropy": 3.9355390071868896, + "top_singular_frac": 0.06085235998034477, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 9.475764628198924, + "spectral_entropy": 3.9166080951690674, + "top_singular_frac": 0.04918518662452698, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 14.05691402391264, + "spectral_entropy": 3.9849467277526855, + "top_singular_frac": 0.03811148926615715, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 11.556752679194709, + "spectral_entropy": 3.925776958465576, + "top_singular_frac": 0.04400862380862236, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 15.187534567823993, + "spectral_entropy": 3.9281375408172607, + "top_singular_frac": 0.038149330765008926, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 15.533853703466942, + "spectral_entropy": 3.9257164001464844, + "top_singular_frac": 0.03785346820950508, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 16.40641267800115, + "spectral_entropy": 3.9651196002960205, + "top_singular_frac": 0.03582756593823433, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 16.752528556677337, + "spectral_entropy": 3.9314327239990234, + "top_singular_frac": 0.03622501716017723, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 10.564028449238647, + "spectral_entropy": 3.92887020111084, + "top_singular_frac": 0.04613117501139641, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 15.057472304610208, + "spectral_entropy": 3.9401333332061768, + "top_singular_frac": 0.03803424909710884, + "rope_alignment": null + } + ] + }, + "20": { + "mean_stable_rank": 12.61066649424165, + "mean_spectral_entropy": 3.9310989528894424, + "mean_top_singular_frac": 0.04354101466014981, + "max_stable_rank_head": 14, + "per_head": [ + { + "head": 0, + "stable_rank": 14.21721269504251, + "spectral_entropy": 3.933781623840332, + "top_singular_frac": 0.0393926240503788, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 4.9088034921185395, + "spectral_entropy": 3.8851065635681152, + "top_singular_frac": 0.07128982990980148, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 7.633927837143474, + "spectral_entropy": 3.9144411087036133, + "top_singular_frac": 0.055547360330820084, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 12.990303210569639, + "spectral_entropy": 3.9424736499786377, + "top_singular_frac": 0.04091976583003998, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 9.979805904421132, + "spectral_entropy": 3.9479105472564697, + "top_singular_frac": 0.04669085144996643, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 14.593654154243286, + "spectral_entropy": 3.918015480041504, + "top_singular_frac": 0.03914165869355202, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.47206403065551, + "spectral_entropy": 3.905104160308838, + "top_singular_frac": 0.04490887001156807, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 13.489652739506717, + "spectral_entropy": 3.932494878768921, + "top_singular_frac": 0.04045813903212547, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 14.13261336052532, + "spectral_entropy": 3.955857276916504, + "top_singular_frac": 0.03877697139978409, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 12.60524463423864, + "spectral_entropy": 3.938253879547119, + "top_singular_frac": 0.041578974574804306, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 15.14339207253686, + "spectral_entropy": 3.9417200088500977, + "top_singular_frac": 0.037891168147325516, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 13.275459756354222, + "spectral_entropy": 3.917692184448242, + "top_singular_frac": 0.04117896035313606, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 15.293006243074993, + "spectral_entropy": 3.9189224243164062, + "top_singular_frac": 0.038227006793022156, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 13.526453994382093, + "spectral_entropy": 3.9333088397979736, + "top_singular_frac": 0.04032106325030327, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 18.393497190964602, + "spectral_entropy": 3.9954445362091064, + "top_singular_frac": 0.033065065741539, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 10.11557259208888, + "spectral_entropy": 3.917056083679199, + "top_singular_frac": 0.04726792499423027, + "rope_alignment": null + } + ] + }, + "21": { + "mean_stable_rank": 12.47517995072221, + "mean_spectral_entropy": 3.934207782149315, + "mean_top_singular_frac": 0.04417073307558894, + "max_stable_rank_head": 5, + "per_head": [ + { + "head": 0, + "stable_rank": 15.146290917621505, + "spectral_entropy": 3.941366195678711, + "top_singular_frac": 0.037902191281318665, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 9.01571008322462, + "spectral_entropy": 3.8784561157226562, + "top_singular_frac": 0.0520990788936615, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 14.333593871981842, + "spectral_entropy": 3.9495296478271484, + "top_singular_frac": 0.038736797869205475, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 8.458295434367447, + "spectral_entropy": 3.9666342735290527, + "top_singular_frac": 0.050325870513916016, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 16.556747900480172, + "spectral_entropy": 3.9619994163513184, + "top_singular_frac": 0.03577782213687897, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 18.588355680520948, + "spectral_entropy": 3.9772911071777344, + "top_singular_frac": 0.033345069736242294, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 8.307249370516624, + "spectral_entropy": 3.9196696281433105, + "top_singular_frac": 0.052784234285354614, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 8.648033106358637, + "spectral_entropy": 3.9258763790130615, + "top_singular_frac": 0.05134936794638634, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 12.101585236648559, + "spectral_entropy": 3.9431614875793457, + "top_singular_frac": 0.04238086938858032, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 14.603753321978877, + "spectral_entropy": 3.9394030570983887, + "top_singular_frac": 0.03875250741839409, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 12.879804409374428, + "spectral_entropy": 3.9382412433624268, + "top_singular_frac": 0.04128052666783333, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 17.54911763205227, + "spectral_entropy": 3.9675283432006836, + "top_singular_frac": 0.034618351608514786, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 8.526068469489465, + "spectral_entropy": 3.8923990726470947, + "top_singular_frac": 0.053046952933073044, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 14.422996847850001, + "spectral_entropy": 3.9525742530822754, + "top_singular_frac": 0.038461778312921524, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 5.684179987579085, + "spectral_entropy": 3.861830472946167, + "top_singular_frac": 0.06709758192300797, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 14.78109694151087, + "spectral_entropy": 3.931363821029663, + "top_singular_frac": 0.03877272829413414, + "rope_alignment": null + } + ] + }, + "22": { + "mean_stable_rank": 10.804104357542, + "mean_spectral_entropy": 3.909366652369499, + "mean_top_singular_frac": 0.05061245302204043, + "max_stable_rank_head": 0, + "per_head": [ + { + "head": 0, + "stable_rank": 20.53885553546018, + "spectral_entropy": 4.006389617919922, + "top_singular_frac": 0.031067458912730217, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 11.71427103252401, + "spectral_entropy": 3.9108848571777344, + "top_singular_frac": 0.04420848190784454, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 2.102186703269165, + "spectral_entropy": 3.850241184234619, + "top_singular_frac": 0.12160985916852951, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 10.066127927251703, + "spectral_entropy": 3.8925602436065674, + "top_singular_frac": 0.04863838106393814, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 11.125711688971363, + "spectral_entropy": 3.902682065963745, + "top_singular_frac": 0.04571553319692612, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 9.569954741756566, + "spectral_entropy": 3.8565454483032227, + "top_singular_frac": 0.051401786506175995, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 12.301340585321618, + "spectral_entropy": 3.9006314277648926, + "top_singular_frac": 0.04352595657110214, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 11.78737045264574, + "spectral_entropy": 3.929992198944092, + "top_singular_frac": 0.04346877709031105, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 10.741971959467227, + "spectral_entropy": 3.9188084602355957, + "top_singular_frac": 0.046091265976428986, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 12.727738426366031, + "spectral_entropy": 3.9441990852355957, + "top_singular_frac": 0.04128147289156914, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 11.391921171771182, + "spectral_entropy": 3.9047675132751465, + "top_singular_frac": 0.04520647972822189, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 10.680311186971355, + "spectral_entropy": 3.92555570602417, + "top_singular_frac": 0.04584285989403725, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 9.388878813837126, + "spectral_entropy": 3.8968894481658936, + "top_singular_frac": 0.05030837655067444, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 7.808727306532336, + "spectral_entropy": 3.912125587463379, + "top_singular_frac": 0.054320987313985825, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 7.996679259742851, + "spectral_entropy": 3.8883185386657715, + "top_singular_frac": 0.05505675449967384, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 12.923622928783535, + "spectral_entropy": 3.9092750549316406, + "top_singular_frac": 0.04205481708049774, + "rope_alignment": null + } + ] + }, + "23": { + "mean_stable_rank": 8.523498701855718, + "mean_spectral_entropy": 3.8589075207710266, + "mean_top_singular_frac": 0.06610549427568913, + "max_stable_rank_head": 1, + "per_head": [ + { + "head": 0, + "stable_rank": 8.464686007484959, + "spectral_entropy": 3.8811206817626953, + "top_singular_frac": 0.053708601742982864, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 13.782500844039305, + "spectral_entropy": 3.956526279449463, + "top_singular_frac": 0.039404209703207016, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 11.276437913052046, + "spectral_entropy": 3.8914103507995605, + "top_singular_frac": 0.04595961421728134, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 13.470149113121728, + "spectral_entropy": 4.000365734100342, + "top_singular_frac": 0.03861844167113304, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 6.20119905132548, + "spectral_entropy": 3.872232437133789, + "top_singular_frac": 0.06395227462053299, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 3.33910832961922, + "spectral_entropy": 3.7580924034118652, + "top_singular_frac": 0.09828423708677292, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 9.456820957978834, + "spectral_entropy": 3.8971285820007324, + "top_singular_frac": 0.04997145012021065, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 10.093656704413046, + "spectral_entropy": 3.889738082885742, + "top_singular_frac": 0.0486740805208683, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 8.058210709449675, + "spectral_entropy": 3.902177333831787, + "top_singular_frac": 0.054160814732313156, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.033222251254431, + "spectral_entropy": 3.852595806121826, + "top_singular_frac": 0.05313989147543907, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.167872594088815, + "spectral_entropy": 3.873858690261841, + "top_singular_frac": 0.05182575434446335, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 8.458857465944837, + "spectral_entropy": 3.894094467163086, + "top_singular_frac": 0.05326766148209572, + "rope_alignment": null + }, + { + "head": 12, + "stable_rank": 10.233085368536981, + "spectral_entropy": 3.8860244750976562, + "top_singular_frac": 0.04844089224934578, + "rope_alignment": null + }, + { + "head": 13, + "stable_rank": 5.560786149676719, + "spectral_entropy": 3.857990264892578, + "top_singular_frac": 0.06862185895442963, + "rope_alignment": null + }, + { + "head": 14, + "stable_rank": 8.480147303017946, + "spectral_entropy": 3.8657894134521484, + "top_singular_frac": 0.054540764540433884, + "rope_alignment": null + }, + { + "head": 15, + "stable_rank": 1.2992384666874557, + "spectral_entropy": 3.4633753299713135, + "top_singular_frac": 0.2351173609495163, + "rope_alignment": null + } + ] + } + } +} \ No newline at end of file diff --git a/data/mi4_svd_wqk/gpt2.json b/data/mi4_svd_wqk/gpt2.json new file mode 100644 index 0000000000000000000000000000000000000000..1a85a592ace627bfbb41ed1c6336113dcd31ba90 --- /dev/null +++ b/data/mi4_svd_wqk/gpt2.json @@ -0,0 +1,1114 @@ +{ + "model": "gpt2", + "N": 12, + "L_crit": null, + "n_heads": 12, + "d_head": 64, + "n_active": null, + "layer_stats": { + "0": { + "mean_stable_rank": 20.54891794294623, + "mean_spectral_entropy": 3.97858460744222, + "mean_top_singular_frac": 0.03878004057332873, + "max_stable_rank_head": 10, + "per_head": [ + { + "head": 0, + "stable_rank": 16.772587903834285, + "spectral_entropy": 3.891568899154663, + "top_singular_frac": 0.037108540534973145, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 25.95965481652204, + "spectral_entropy": 4.050695419311523, + "top_singular_frac": 0.026456860825419426, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 14.228952302908226, + "spectral_entropy": 3.902189016342163, + "top_singular_frac": 0.040134843438863754, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 23.39769034803083, + "spectral_entropy": 4.084882736206055, + "top_singular_frac": 0.02758875861763954, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 22.28319199036109, + "spectral_entropy": 4.074897766113281, + "top_singular_frac": 0.02853013575077057, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 31.514598281841884, + "spectral_entropy": 4.082849502563477, + "top_singular_frac": 0.023471342399716377, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 5.8548667665825835, + "spectral_entropy": 3.8688101768493652, + "top_singular_frac": 0.06529883295297623, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 20.271938306949398, + "spectral_entropy": 4.01046895980835, + "top_singular_frac": 0.03121809847652912, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 16.874305450544085, + "spectral_entropy": 3.9101405143737793, + "top_singular_frac": 0.03658284619450569, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 28.51838044894121, + "spectral_entropy": 4.045000076293945, + "top_singular_frac": 0.025383008643984795, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 37.74390616825879, + "spectral_entropy": 4.11109733581543, + "top_singular_frac": 0.021044833585619926, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 3.1669425305803394, + "spectral_entropy": 3.7104148864746094, + "top_singular_frac": 0.10254238545894623, + "rope_alignment": null + } + ] + }, + "1": { + "mean_stable_rank": 5.408035022885328, + "mean_spectral_entropy": 3.523526887098948, + "mean_top_singular_frac": 0.10943823866546154, + "max_stable_rank_head": 5, + "per_head": [ + { + "head": 0, + "stable_rank": 6.741789385862621, + "spectral_entropy": 3.927548408508301, + "top_singular_frac": 0.058860644698143005, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 4.912148493144858, + "spectral_entropy": 3.8125648498535156, + "top_singular_frac": 0.0784454494714737, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 2.0761865263019037, + "spectral_entropy": 2.965096950531006, + "top_singular_frac": 0.2135297656059265, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 2.707892599177071, + "spectral_entropy": 2.8835039138793945, + "top_singular_frac": 0.19209466874599457, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 2.8274264246558167, + "spectral_entropy": 3.1331522464752197, + "top_singular_frac": 0.1634136587381363, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 12.699378115222647, + "spectral_entropy": 3.9372670650482178, + "top_singular_frac": 0.04170381277799606, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 7.927121733790247, + "spectral_entropy": 3.8810224533081055, + "top_singular_frac": 0.05514828860759735, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 6.506350138531822, + "spectral_entropy": 3.9508345127105713, + "top_singular_frac": 0.058792535215616226, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 4.957943308873153, + "spectral_entropy": 3.3704967498779297, + "top_singular_frac": 0.09863679111003876, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 3.2391695065215416, + "spectral_entropy": 3.44173526763916, + "top_singular_frac": 0.12059128284454346, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 2.9336436658876712, + "spectral_entropy": 2.9010934829711914, + "top_singular_frac": 0.18215598165988922, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 7.367370376654579, + "spectral_entropy": 4.078006744384766, + "top_singular_frac": 0.049885984510183334, + "rope_alignment": null + } + ] + }, + "2": { + "mean_stable_rank": 2.9981938292263277, + "mean_spectral_entropy": 3.548010249932607, + "mean_top_singular_frac": 0.14282788274188837, + "max_stable_rank_head": 6, + "per_head": [ + { + "head": 0, + "stable_rank": 3.306441829390279, + "spectral_entropy": 3.748694896697998, + "top_singular_frac": 0.10423664003610611, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 5.899783138667421, + "spectral_entropy": 3.85343861579895, + "top_singular_frac": 0.06669148057699203, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 2.0174223599061762, + "spectral_entropy": 3.445655345916748, + "top_singular_frac": 0.18624968826770782, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 2.142010877003213, + "spectral_entropy": 3.5436182022094727, + "top_singular_frac": 0.1642923653125763, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 2.7519354311336497, + "spectral_entropy": 3.6701486110687256, + "top_singular_frac": 0.12455720454454422, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 2.0065955749831863, + "spectral_entropy": 3.4051661491394043, + "top_singular_frac": 0.19114762544631958, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 5.95398400109017, + "spectral_entropy": 3.710178852081299, + "top_singular_frac": 0.07370765507221222, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 1.867614619133873, + "spectral_entropy": 3.2985708713531494, + "top_singular_frac": 0.1951521635055542, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 2.418668100968061, + "spectral_entropy": 3.6216399669647217, + "top_singular_frac": 0.14443325996398926, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 2.4169053741147897, + "spectral_entropy": 3.5364794731140137, + "top_singular_frac": 0.1579783856868744, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 2.750753047183869, + "spectral_entropy": 3.3745944499969482, + "top_singular_frac": 0.14768365025520325, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 2.44621159714124, + "spectral_entropy": 3.3679375648498535, + "top_singular_frac": 0.157804474234581, + "rope_alignment": null + } + ] + }, + "3": { + "mean_stable_rank": 9.839873432042834, + "mean_spectral_entropy": 3.922416925430298, + "mean_top_singular_frac": 0.05075060483068228, + "max_stable_rank_head": 9, + "per_head": [ + { + "head": 0, + "stable_rank": 5.449403846858111, + "spectral_entropy": 3.9531264305114746, + "top_singular_frac": 0.06530832499265671, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 13.562461754989188, + "spectral_entropy": 3.9443273544311523, + "top_singular_frac": 0.040346067398786545, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 7.00235515207363, + "spectral_entropy": 3.8598008155822754, + "top_singular_frac": 0.06154673919081688, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.61780438950167, + "spectral_entropy": 3.8996005058288574, + "top_singular_frac": 0.056831177324056625, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 12.31611151004516, + "spectral_entropy": 3.9858994483947754, + "top_singular_frac": 0.04096503555774689, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 12.012644122151606, + "spectral_entropy": 3.946624755859375, + "top_singular_frac": 0.04275018721818924, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 7.479484687179621, + "spectral_entropy": 3.874399185180664, + "top_singular_frac": 0.0580713152885437, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 7.1967249712706955, + "spectral_entropy": 3.929482936859131, + "top_singular_frac": 0.05770816653966904, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 7.07705202251314, + "spectral_entropy": 3.9308109283447266, + "top_singular_frac": 0.057764697819948196, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 14.633598180230253, + "spectral_entropy": 3.929500102996826, + "top_singular_frac": 0.03891022130846977, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 13.43906103644516, + "spectral_entropy": 3.8707034587860107, + "top_singular_frac": 0.0422184020280838, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 10.2917795112558, + "spectral_entropy": 3.9447271823883057, + "top_singular_frac": 0.04658692330121994, + "rope_alignment": null + } + ] + }, + "4": { + "mean_stable_rank": 10.721275175771977, + "mean_spectral_entropy": 3.912612239519755, + "mean_top_singular_frac": 0.04788041456292073, + "max_stable_rank_head": 4, + "per_head": [ + { + "head": 0, + "stable_rank": 8.913847300362841, + "spectral_entropy": 3.86849308013916, + "top_singular_frac": 0.05322720482945442, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 10.17227629640135, + "spectral_entropy": 3.910954236984253, + "top_singular_frac": 0.04793316125869751, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 8.90250239087533, + "spectral_entropy": 3.9107863903045654, + "top_singular_frac": 0.05136754363775253, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 10.809024798176072, + "spectral_entropy": 3.889604330062866, + "top_singular_frac": 0.04699653014540672, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 15.46201226219978, + "spectral_entropy": 3.9376115798950195, + "top_singular_frac": 0.03761986643075943, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 10.740366184261916, + "spectral_entropy": 3.8961727619171143, + "top_singular_frac": 0.04705396667122841, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.845440222816366, + "spectral_entropy": 3.9918618202209473, + "top_singular_frac": 0.04162279888987541, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 13.276492322130073, + "spectral_entropy": 3.9776597023010254, + "top_singular_frac": 0.039631765335798264, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 8.523950149114564, + "spectral_entropy": 3.8947322368621826, + "top_singular_frac": 0.05289027467370033, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 15.434901394094016, + "spectral_entropy": 3.9583823680877686, + "top_singular_frac": 0.03709797561168671, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 6.815730461493818, + "spectral_entropy": 3.897758960723877, + "top_singular_frac": 0.05915495753288269, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 7.758758327337592, + "spectral_entropy": 3.8173294067382812, + "top_singular_frac": 0.05996892973780632, + "rope_alignment": null + } + ] + }, + "5": { + "mean_stable_rank": 13.397514340774647, + "mean_spectral_entropy": 3.930776576201121, + "mean_top_singular_frac": 0.041944171922902264, + "max_stable_rank_head": 1, + "per_head": [ + { + "head": 0, + "stable_rank": 12.499260733013466, + "spectral_entropy": 3.8399033546447754, + "top_singular_frac": 0.04469495639204979, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 18.231113922801015, + "spectral_entropy": 3.9823131561279297, + "top_singular_frac": 0.03350158408284187, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 11.792627459337753, + "spectral_entropy": 3.9038329124450684, + "top_singular_frac": 0.04441992565989494, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 10.031812854775813, + "spectral_entropy": 3.9716761112213135, + "top_singular_frac": 0.04597597196698189, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 8.8100196142071, + "spectral_entropy": 3.8733649253845215, + "top_singular_frac": 0.05302615091204643, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 17.239357698343944, + "spectral_entropy": 3.9256327152252197, + "top_singular_frac": 0.03581751883029938, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 15.395531245216064, + "spectral_entropy": 3.9918556213378906, + "top_singular_frac": 0.036198802292346954, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 14.315359434978093, + "spectral_entropy": 3.9171035289764404, + "top_singular_frac": 0.03969641029834747, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 14.31262862359247, + "spectral_entropy": 3.889373302459717, + "top_singular_frac": 0.0405779629945755, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 13.529798863266253, + "spectral_entropy": 4.002249240875244, + "top_singular_frac": 0.0385473407804966, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 7.624617578569405, + "spectral_entropy": 3.900118589401245, + "top_singular_frac": 0.05574554204940796, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 16.988044061194397, + "spectral_entropy": 3.971895456314087, + "top_singular_frac": 0.035127896815538406, + "rope_alignment": null + } + ] + }, + "6": { + "mean_stable_rank": 10.385380729293983, + "mean_spectral_entropy": 3.8834681709607444, + "mean_top_singular_frac": 0.05067484204967817, + "max_stable_rank_head": 9, + "per_head": [ + { + "head": 0, + "stable_rank": 10.493835308918278, + "spectral_entropy": 3.87605619430542, + "top_singular_frac": 0.048185236752033234, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 12.288235206129885, + "spectral_entropy": 3.9089105129241943, + "top_singular_frac": 0.043359722942113876, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 9.861276550430194, + "spectral_entropy": 3.8827483654022217, + "top_singular_frac": 0.049483560025691986, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 12.212110363160614, + "spectral_entropy": 3.953441619873047, + "top_singular_frac": 0.04202663525938988, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 8.237445353936831, + "spectral_entropy": 3.883042097091675, + "top_singular_frac": 0.05437833443284035, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 7.40615387605852, + "spectral_entropy": 3.8541159629821777, + "top_singular_frac": 0.059031132608652115, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 15.84816259608988, + "spectral_entropy": 3.9643642902374268, + "top_singular_frac": 0.036519892513751984, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 10.010067817406421, + "spectral_entropy": 3.929054021835327, + "top_singular_frac": 0.047457944601774216, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 5.8782343878074315, + "spectral_entropy": 3.7356553077697754, + "top_singular_frac": 0.07297303527593613, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 17.442705874675, + "spectral_entropy": 3.941126823425293, + "top_singular_frac": 0.03530983254313469, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 6.772004575042436, + "spectral_entropy": 3.826209545135498, + "top_singular_frac": 0.06285222619771957, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 8.17433684187231, + "spectral_entropy": 3.846893310546875, + "top_singular_frac": 0.056520551443099976, + "rope_alignment": null + } + ] + }, + "7": { + "mean_stable_rank": 10.130267363522004, + "mean_spectral_entropy": 3.8654574155807495, + "mean_top_singular_frac": 0.052597637909154095, + "max_stable_rank_head": 2, + "per_head": [ + { + "head": 0, + "stable_rank": 5.210730454141143, + "spectral_entropy": 3.737128257751465, + "top_singular_frac": 0.0797845721244812, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 9.865775000441863, + "spectral_entropy": 3.8715033531188965, + "top_singular_frac": 0.04969180002808571, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 15.905931163226311, + "spectral_entropy": 3.913206100463867, + "top_singular_frac": 0.037916842848062515, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.031114381829853, + "spectral_entropy": 3.8617441654205322, + "top_singular_frac": 0.061126116663217545, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 7.7627874438176345, + "spectral_entropy": 3.8367743492126465, + "top_singular_frac": 0.05827735364437103, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 11.834289504409465, + "spectral_entropy": 3.9347238540649414, + "top_singular_frac": 0.04330651834607124, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.408370017900852, + "spectral_entropy": 3.9280898571014404, + "top_singular_frac": 0.04426136985421181, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 12.013417672964428, + "spectral_entropy": 3.8746418952941895, + "top_singular_frac": 0.044802695512771606, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 7.101915322421357, + "spectral_entropy": 3.8180108070373535, + "top_singular_frac": 0.06276211887598038, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 6.4536232302721155, + "spectral_entropy": 3.8164315223693848, + "top_singular_frac": 0.06626778095960617, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 13.168109802964215, + "spectral_entropy": 3.913646697998047, + "top_singular_frac": 0.041452985256910324, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 13.807144367874807, + "spectral_entropy": 3.8795881271362305, + "top_singular_frac": 0.041521500796079636, + "rope_alignment": null + } + ] + }, + "8": { + "mean_stable_rank": 8.588102707164909, + "mean_spectral_entropy": 3.849526286125183, + "mean_top_singular_frac": 0.06226382311433554, + "max_stable_rank_head": 1, + "per_head": [ + { + "head": 0, + "stable_rank": 13.815290682213616, + "spectral_entropy": 3.928388833999634, + "top_singular_frac": 0.04022029787302017, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 15.726194469400188, + "spectral_entropy": 3.949498414993286, + "top_singular_frac": 0.03716174513101578, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 3.1238252788084044, + "spectral_entropy": 3.7848122119903564, + "top_singular_frac": 0.09919799864292145, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 10.796249397346587, + "spectral_entropy": 3.916820526123047, + "top_singular_frac": 0.0458897165954113, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 5.335745521056488, + "spectral_entropy": 3.732435464859009, + "top_singular_frac": 0.07991687953472137, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 4.6177851155437155, + "spectral_entropy": 3.699676036834717, + "top_singular_frac": 0.08782633394002914, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 13.736777877307071, + "spectral_entropy": 3.937767505645752, + "top_singular_frac": 0.03998145833611488, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 4.51364863846439, + "spectral_entropy": 3.619126558303833, + "top_singular_frac": 0.0964168831706047, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 7.821900576496092, + "spectral_entropy": 3.8958168029785156, + "top_singular_frac": 0.05545078217983246, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 9.144724399697461, + "spectral_entropy": 3.9262747764587402, + "top_singular_frac": 0.049821872264146805, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 7.904858234053957, + "spectral_entropy": 3.935814142227173, + "top_singular_frac": 0.05343753844499588, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 6.520232295590924, + "spectral_entropy": 3.8678841590881348, + "top_singular_frac": 0.061844371259212494, + "rope_alignment": null + } + ] + }, + "9": { + "mean_stable_rank": 10.42315899068788, + "mean_spectral_entropy": 3.852445125579834, + "mean_top_singular_frac": 0.05871348517636458, + "max_stable_rank_head": 5, + "per_head": [ + { + "head": 0, + "stable_rank": 11.765789759193764, + "spectral_entropy": 3.9414262771606445, + "top_singular_frac": 0.04304874688386917, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 15.305155734609425, + "spectral_entropy": 3.9585299491882324, + "top_singular_frac": 0.037246573716402054, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 10.230307747362096, + "spectral_entropy": 3.91330885887146, + "top_singular_frac": 0.047461435198783875, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 4.058931607314408, + "spectral_entropy": 3.5093493461608887, + "top_singular_frac": 0.11280500888824463, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 12.221413314786355, + "spectral_entropy": 3.935237407684326, + "top_singular_frac": 0.042423609644174576, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 16.716316241355518, + "spectral_entropy": 3.993741273880005, + "top_singular_frac": 0.03478403016924858, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 14.327458503403285, + "spectral_entropy": 3.942858934402466, + "top_singular_frac": 0.03885767608880997, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 3.3482205795352513, + "spectral_entropy": 3.7969913482666016, + "top_singular_frac": 0.09526665508747101, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 8.74447781232857, + "spectral_entropy": 3.8921151161193848, + "top_singular_frac": 0.0522313229739666, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 16.28833757151712, + "spectral_entropy": 3.956057071685791, + "top_singular_frac": 0.03622189909219742, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 4.201228750622032, + "spectral_entropy": 3.5410940647125244, + "top_singular_frac": 0.1073368713259697, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 7.870270266226736, + "spectral_entropy": 3.8486318588256836, + "top_singular_frac": 0.056877993047237396, + "rope_alignment": null + } + ] + }, + "10": { + "mean_stable_rank": 9.766436836913453, + "mean_spectral_entropy": 3.8401251832644143, + "mean_top_singular_frac": 0.05966843261073033, + "max_stable_rank_head": 8, + "per_head": [ + { + "head": 0, + "stable_rank": 7.787827189587669, + "spectral_entropy": 3.879901885986328, + "top_singular_frac": 0.05634399503469467, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 12.28844300101696, + "spectral_entropy": 3.9502358436584473, + "top_singular_frac": 0.041923750191926956, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 14.826304571414397, + "spectral_entropy": 3.919924736022949, + "top_singular_frac": 0.03889035806059837, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 7.470295062749167, + "spectral_entropy": 3.916905403137207, + "top_singular_frac": 0.05535682663321495, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 7.7879825939991605, + "spectral_entropy": 3.8939003944396973, + "top_singular_frac": 0.05543794482946396, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 4.087582821704354, + "spectral_entropy": 3.4018325805664062, + "top_singular_frac": 0.12106303870677948, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 11.820122696802231, + "spectral_entropy": 3.925790786743164, + "top_singular_frac": 0.04353642463684082, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 13.316274670840079, + "spectral_entropy": 3.9559335708618164, + "top_singular_frac": 0.04037044197320938, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 17.180158722134625, + "spectral_entropy": 3.9715254306793213, + "top_singular_frac": 0.03476087376475334, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 3.899515432019366, + "spectral_entropy": 3.401111125946045, + "top_singular_frac": 0.12437739968299866, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 9.20681444045566, + "spectral_entropy": 3.9301352500915527, + "top_singular_frac": 0.0494387187063694, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 7.52592084023777, + "spectral_entropy": 3.934305191040039, + "top_singular_frac": 0.05452141910791397, + "rope_alignment": null + } + ] + }, + "11": { + "mean_stable_rank": 6.929702107275941, + "mean_spectral_entropy": 3.8606763084729514, + "mean_top_singular_frac": 0.0667615169659257, + "max_stable_rank_head": 7, + "per_head": [ + { + "head": 0, + "stable_rank": 6.932838815581778, + "spectral_entropy": 3.885451316833496, + "top_singular_frac": 0.059482503682374954, + "rope_alignment": null + }, + { + "head": 1, + "stable_rank": 10.631444741769325, + "spectral_entropy": 3.8975844383239746, + "top_singular_frac": 0.047052085399627686, + "rope_alignment": null + }, + { + "head": 2, + "stable_rank": 6.872039949897464, + "spectral_entropy": 3.8743844032287598, + "top_singular_frac": 0.06043419614434242, + "rope_alignment": null + }, + { + "head": 3, + "stable_rank": 3.932279224420238, + "spectral_entropy": 3.8263792991638184, + "top_singular_frac": 0.08522184193134308, + "rope_alignment": null + }, + { + "head": 4, + "stable_rank": 3.292454816072897, + "spectral_entropy": 3.819840908050537, + "top_singular_frac": 0.09381072968244553, + "rope_alignment": null + }, + { + "head": 5, + "stable_rank": 9.088192813772363, + "spectral_entropy": 3.915809154510498, + "top_singular_frac": 0.05038663372397423, + "rope_alignment": null + }, + { + "head": 6, + "stable_rank": 9.068627276109853, + "spectral_entropy": 3.8489179611206055, + "top_singular_frac": 0.05285482481122017, + "rope_alignment": null + }, + { + "head": 7, + "stable_rank": 10.93768566096611, + "spectral_entropy": 3.916231632232666, + "top_singular_frac": 0.04567831754684448, + "rope_alignment": null + }, + { + "head": 8, + "stable_rank": 2.49842332153342, + "spectral_entropy": 3.7718639373779297, + "top_singular_frac": 0.11428188532590866, + "rope_alignment": null + }, + { + "head": 9, + "stable_rank": 8.83917901209052, + "spectral_entropy": 3.92108154296875, + "top_singular_frac": 0.05084900185465813, + "rope_alignment": null + }, + { + "head": 10, + "stable_rank": 5.8049308631137615, + "spectral_entropy": 3.794830083847046, + "top_singular_frac": 0.07052602618932724, + "rope_alignment": null + }, + { + "head": 11, + "stable_rank": 5.258328791983553, + "spectral_entropy": 3.855741024017334, + "top_singular_frac": 0.07056015729904175, + "rope_alignment": null + } + ] + } + } +} \ No newline at end of file diff --git a/data/mi5_coop/gpt2-medium.json b/data/mi5_coop/gpt2-medium.json new file mode 100644 index 0000000000000000000000000000000000000000..d2860f7ae365c82a83d0a90572e8a0af028e5a3d --- /dev/null +++ b/data/mi5_coop/gpt2-medium.json @@ -0,0 +1,351 @@ +{ + "cos_sim_mean": -0.01749146729707718, + "cos_sim_std": 0.11474757641553879, + "cos_sim_matrix": [ + [ + 1.0, + -0.029719023033976555, + -0.020083043724298477, + -0.17210397124290466, + 0.06633961945772171, + -0.030295822769403458, + 0.055486612021923065, + -0.017657075077295303, + -0.02941712737083435, + -0.015339821577072144, + 0.01223018765449524, + -0.006009957753121853, + 0.04284694790840149, + 0.04262711852788925, + 0.02499740943312645, + -0.05745357275009155 + ], + [ + -0.029719023033976555, + 1.0, + -0.019175421446561813, + 0.09524994343519211, + -0.10856646299362183, + 0.10407213866710663, + -0.13017474114894867, + 0.07561583817005157, + 0.0619095042347908, + 0.07305620610713959, + -0.01670997589826584, + 0.02275921031832695, + -0.060057613998651505, + 0.006176801398396492, + -0.011007066816091537, + -0.0018594367429614067 + ], + [ + -0.020083043724298477, + -0.019175421446561813, + 0.9999999403953552, + -0.13182584941387177, + 0.1573365330696106, + -0.055456504225730896, + -0.09673528373241425, + -0.2845197319984436, + 0.03393082320690155, + -0.00966035109013319, + -0.10627517104148865, + 0.09607540816068649, + 0.0007109753787517548, + 0.00547989085316658, + 0.023064905777573586, + 0.012807110324501991 + ], + [ + -0.17210397124290466, + 0.09524994343519211, + -0.13182584941387177, + 0.9999998807907104, + -0.16175492107868195, + 0.24149717390537262, + 0.054623670876026154, + 0.28946176171302795, + 0.08893273770809174, + 0.030161606147885323, + 0.13382771611213684, + -0.07488537579774857, + 0.006063426379114389, + 0.005872063804417849, + -0.08418633788824081, + -0.1655772179365158 + ], + [ + 0.06633961945772171, + -0.10856646299362183, + 0.1573365330696106, + -0.16175492107868195, + 1.0000001192092896, + -0.2774767577648163, + 0.08340868353843689, + -0.13156937062740326, + -0.23705127835273743, + -0.09026273339986801, + -0.11047787964344025, + -0.05610831454396248, + -0.06804681569337845, + -0.10817133635282516, + -0.026060866191983223, + 0.07559887319803238 + ], + [ + -0.030295822769403458, + 0.10407213866710663, + -0.055456504225730896, + 0.24149717390537262, + -0.2774767577648163, + 1.0, + -0.35720205307006836, + 0.26169487833976746, + 0.2187243551015854, + -0.007805807515978813, + 0.0708412379026413, + 0.0027274591848254204, + 0.006418980658054352, + 0.08051083981990814, + 0.04310116171836853, + -0.5057379603385925 + ], + [ + 0.055486612021923065, + -0.13017474114894867, + -0.09673528373241425, + 0.054623670876026154, + 0.08340868353843689, + -0.35720205307006836, + 1.0, + 0.039921291172504425, + -0.2335905283689499, + -0.15781259536743164, + 0.007157748565077782, + -0.032690949738025665, + -0.018237994983792305, + -0.09058814495801926, + -0.027706000953912735, + -0.18492776155471802 + ], + [ + -0.017657075077295303, + 0.07561583817005157, + -0.2845197319984436, + 0.28946176171302795, + -0.13156937062740326, + 0.26169487833976746, + 0.039921291172504425, + 1.0000001192092896, + 0.09676885604858398, + -0.014062846079468727, + 0.1872798204421997, + -0.06640951335430145, + -0.003437318606302142, + 0.04130113869905472, + 0.03180164471268654, + -0.30804750323295593 + ], + [ + -0.02941712737083435, + 0.0619095042347908, + 0.03393082320690155, + 0.08893273770809174, + -0.23705127835273743, + 0.2187243551015854, + -0.2335905283689499, + 0.09676885604858398, + 1.0, + 0.012450695037841797, + 0.05234065651893616, + -0.10017211735248566, + 0.07453452795743942, + 0.0031156991608440876, + 0.02152005396783352, + -0.11511378735303879 + ], + [ + -0.015339821577072144, + 0.07305620610713959, + -0.00966035109013319, + 0.030161606147885323, + -0.09026273339986801, + -0.007805807515978813, + -0.15781259536743164, + -0.014062846079468727, + 0.012450695037841797, + 1.0000001192092896, + -0.036916568875312805, + 0.03244571015238762, + 0.020190328359603882, + 0.006737357005476952, + 0.06574798375368118, + 0.10236559063196182 + ], + [ + 0.01223018765449524, + -0.01670997589826584, + -0.10627517104148865, + 0.13382771611213684, + -0.11047787964344025, + 0.0708412379026413, + 0.007157748565077782, + 0.1872798204421997, + 0.05234065651893616, + -0.036916568875312805, + 0.9999998807907104, + -0.07314002513885498, + 0.012890877202153206, + -0.011475431732833385, + 0.008203467354178429, + -0.09418699890375137 + ], + [ + -0.006009957753121853, + 0.02275921031832695, + 0.09607540816068649, + -0.07488537579774857, + -0.05610831454396248, + 0.0027274591848254204, + -0.032690949738025665, + -0.06640951335430145, + -0.10017211735248566, + 0.03244571015238762, + -0.07314002513885498, + 1.0000001192092896, + -0.0012570242397487164, + -0.17616260051727295, + 0.008794736117124557, + 0.047377318143844604 + ], + [ + 0.04284694790840149, + -0.060057613998651505, + 0.0007109753787517548, + 0.006063426379114389, + -0.06804681569337845, + 0.006418980658054352, + -0.018237994983792305, + -0.003437318606302142, + 0.07453452795743942, + 0.020190328359603882, + 0.012890877202153206, + -0.0012570242397487164, + 1.0, + -0.021525036543607712, + 0.09623224288225174, + -0.049355436116456985 + ], + [ + 0.04262711852788925, + 0.006176801398396492, + 0.00547989085316658, + 0.005872063804417849, + -0.10817133635282516, + 0.08051083981990814, + -0.09058814495801926, + 0.04130113869905472, + 0.0031156991608440876, + 0.006737357005476952, + -0.011475431732833385, + -0.17616260051727295, + -0.021525036543607712, + 1.0, + 0.004040432162582874, + -0.05991571024060249 + ], + [ + 0.02499740943312645, + -0.011007066816091537, + 0.023064905777573586, + -0.08418633788824081, + -0.026060866191983223, + 0.04310116171836853, + -0.027706000953912735, + 0.03180164471268654, + 0.02152005396783352, + 0.06574798375368118, + 0.008203467354178429, + 0.008794736117124557, + 0.09623224288225174, + 0.004040432162582874, + 1.0, + -0.029224151745438576 + ], + [ + -0.05745357275009155, + -0.0018594367429614067, + 0.012807110324501991, + -0.1655772179365158, + 0.07559887319803238, + -0.5057379603385925, + -0.18492776155471802, + -0.30804750323295593, + -0.11511378735303879, + 0.10236559063196182, + -0.09418699890375137, + 0.047377318143844604, + -0.049355436116456985, + -0.05991571024060249, + -0.029224151745438576, + 1.0 + ] + ], + "W_sum_frob": 437.4562683105469, + "W_sum_SR": 2.955085841591785, + "W_sum_spec": 254.47763061523438, + "sum_frob_ind": 1795.0878829956055, + "frob_ratio": 0.24369629612814772, + "R_joint": { + "mean": -0.07880064846205406, + "std": 0.29822166072155476, + "all": [ + 0.06742614791549136, + 0.0976621204683245, + 0.05938054008873661, + -0.29952072152409265, + -0.17265655210422787, + -0.8168725465888622, + 0.2853584846216922, + -0.1979649827695418, + 0.029288318224557453, + 0.15989270704738165 + ] + }, + "R_sum_ind": { + "mean": -0.06547276211242853, + "std": 0.2898316977544356, + "all": [ + 0.06953390955019846, + 0.08061531377348458, + 0.0492499641799944, + -0.2988063088272202, + -0.08820839148435931, + -0.8140479822180969, + 0.2735786875915331, + -0.10542988282495344, + 0.028367354790717027, + 0.15041971434441706 + ] + }, + "ratio": 1.2035640158632575, + "interpretation": "ADITIVO \u2014 cabezas independientes", + "per_head_mean": { + "0": 0.018986892453696422, + "1": 0.017546445909191622, + "2": -0.015217878550992525, + "3": 0.005770748801129926, + "4": 0.0287910256636168, + "5": -0.037733797134708966, + "6": 0.0014245624388056666, + "7": 0.0029230501848608336, + "8": -0.035300772987106446, + "9": 0.003711442698753642, + "10": -0.0013832196629693385, + "11": -0.006857158187460829, + "12": -0.003541348947449701, + "13": -0.0053926913484132625, + "14": -0.0024551699644921035, + "15": -0.03674489347889025 + } +} \ No newline at end of file diff --git a/data/mi5_coop/gpt2.json b/data/mi5_coop/gpt2.json new file mode 100644 index 0000000000000000000000000000000000000000..748f43282b3f4035063ab60c08b71be2e17facd7 --- /dev/null +++ b/data/mi5_coop/gpt2.json @@ -0,0 +1,225 @@ +{ + "cos_sim_mean": -0.006194185931235552, + "cos_sim_std": 0.08916603773832321, + "cos_sim_matrix": [ + [ + 0.9999999403953552, + -0.0781468078494072, + 0.19044063985347748, + 0.012521451339125633, + -0.17482100427150726, + 0.05868000164628029, + -0.13370949029922485, + -0.08185403794050217, + -0.1450050324201584, + -0.011590509675443172, + -0.12377012521028519, + -0.11165498197078705 + ], + [ + -0.0781468078494072, + 0.9999999403953552, + -0.09003380686044693, + -0.011253136210143566, + 0.23454049229621887, + 0.017756205052137375, + -0.013420753180980682, + 0.0701494887471199, + 0.05572107434272766, + -0.014212110079824924, + -0.05710172280669212, + 0.010240965522825718 + ], + [ + 0.19044063985347748, + -0.09003380686044693, + 1.0000001192092896, + 0.03555312380194664, + 0.03215237334370613, + 0.033454447984695435, + -0.09831168502569199, + -0.022438479587435722, + -0.030973993241786957, + 0.01098974421620369, + -0.0787498727440834, + 0.0018108654767274857 + ], + [ + 0.012521451339125633, + -0.011253136210143566, + 0.03555312380194664, + 1.0000001192092896, + -0.037501316517591476, + 0.04056897759437561, + -0.046119119971990585, + -0.005569756031036377, + -0.03817259520292282, + 0.052768561989068985, + -0.03212574124336243, + 0.009138870984315872 + ], + [ + -0.17482100427150726, + 0.23454049229621887, + 0.03215237334370613, + -0.037501316517591476, + 1.0000001192092896, + -0.06983646005392075, + -0.22654560208320618, + -0.010897491127252579, + 0.03826494142413139, + -0.09748274832963943, + -0.07146253436803818, + 0.28340578079223633 + ], + [ + 0.05868000164628029, + 0.017756205052137375, + 0.033454447984695435, + 0.04056897759437561, + -0.06983646005392075, + 1.0, + -0.07986623048782349, + 0.017307065427303314, + -0.03909045830368996, + 9.178416803479195e-05, + -0.08784979581832886, + 0.058840926736593246 + ], + [ + -0.13370949029922485, + -0.013420753180980682, + -0.09831168502569199, + -0.046119119971990585, + -0.22654560208320618, + -0.07986623048782349, + 1.0, + 0.051483094692230225, + 0.024282187223434448, + 0.060412466526031494, + 0.14752785861492157, + -0.050319552421569824 + ], + [ + -0.08185403794050217, + 0.0701494887471199, + -0.022438479587435722, + -0.005569756031036377, + -0.010897491127252579, + 0.017307065427303314, + 0.051483094692230225, + 1.0000001192092896, + -0.020404847338795662, + 0.13178551197052002, + 0.14258556067943573, + -0.0009097880683839321 + ], + [ + -0.1450050324201584, + 0.05572107434272766, + -0.030973993241786957, + -0.03817259520292282, + 0.03826494142413139, + -0.03909045830368996, + 0.024282187223434448, + -0.020404847338795662, + 1.0000001192092896, + -0.0672520250082016, + -0.03312409296631813, + -0.026871640235185623 + ], + [ + -0.011590509675443172, + -0.014212110079824924, + 0.01098974421620369, + 0.052768561989068985, + -0.09748274832963943, + 9.178416803479195e-05, + 0.060412466526031494, + 0.13178551197052002, + -0.0672520250082016, + 1.0000001192092896, + 0.13087090849876404, + 0.026185454800724983 + ], + [ + -0.12377012521028519, + -0.05710172280669212, + -0.0787498727440834, + -0.03212574124336243, + -0.07146253436803818, + -0.08784979581832886, + 0.14752785861492157, + 0.14258556067943573, + -0.03312409296631813, + 0.13087090849876404, + 0.9999999403953552, + -0.06989780813455582 + ], + [ + -0.11165498197078705, + 0.010240965522825718, + 0.0018108654767274857, + 0.009138870984315872, + 0.28340578079223633, + 0.058840926736593246, + -0.050319552421569824, + -0.0009097880683839321, + -0.026871640235185623, + 0.026185454800724983, + -0.06989780813455582, + 1.0000001192092896 + ] + ], + "W_sum_frob": 416.15228271484375, + "W_sum_SR": 1.510416012268994, + "W_sum_spec": 338.61328125, + "sum_frob_ind": 1261.7138481140137, + "frob_ratio": 0.3298309544092747, + "R_joint": { + "mean": 0.38012702234507106, + "std": 1.7965894336802049, + "all": [ + 0.17246828302521525, + 0.15637507722670843, + -0.33523551553273556, + -0.5779238863697599, + -1.0560486617584126, + 0.222061666345828, + -0.2776140532081165, + -0.2267634868772568, + 5.343823778254169 + ] + }, + "R_sum_ind": { + "mean": 0.4052737739888028, + "std": 1.8290119442101296, + "all": [ + 0.18572527882060696, + 0.15478360863705976, + -0.294052560283108, + -0.5833130506493405, + -0.9906174917477094, + 0.2386759084635155, + -0.3232760489494515, + -0.20981201969303667, + 5.469350341300689 + ] + }, + "ratio": 0.937951176125356, + "interpretation": "ADITIVO \u2014 cabezas independientes", + "per_head_mean": { + "0": 0.016118080197960634, + "1": 0.054666809158802426, + "2": 0.03426808662852743, + "3": -0.07214545839065195, + "4": -0.04473915585821595, + "5": 0.13960691983726414, + "6": 0.01875832422755956, + "7": -0.03865611482149485, + "8": 0.20553891115197173, + "9": 0.030384311278240976, + "10": -0.0033342157435587716, + "11": 0.06480727632239751 + } +} \ No newline at end of file diff --git a/data/mi5_logit_lens/gpt2-large.json b/data/mi5_logit_lens/gpt2-large.json new file mode 100644 index 0000000000000000000000000000000000000000..0b34439d4ae7d789eb5655f198dc72b55f612d50 --- /dev/null +++ b/data/mi5_logit_lens/gpt2-large.json @@ -0,0 +1,259 @@ +{ + "model": "gpt2-large", + "N": 36, + "L_crit": 33, + "summary": { + "0": { + "rank_median": 295.0, + "rank_mean": 1844.3, + "prob_mean": 0.0008142976017865067, + "entropy_mean": 9.040202701091767, + "top1_acc": 0.0 + }, + "1": { + "rank_median": 168.0, + "rank_mean": 1519.5, + "prob_mean": 0.00437108885781754, + "entropy_mean": 7.437587869167328, + "top1_acc": 0.0 + }, + "2": { + "rank_median": 81.0, + "rank_mean": 1509.35, + "prob_mean": 0.022237656241270543, + "entropy_mean": 5.999598908424377, + "top1_acc": 0.0 + }, + "3": { + "rank_median": 58.5, + "rank_mean": 1388.65, + "prob_mean": 0.022854797989299413, + "entropy_mean": 5.4002340450882915, + "top1_acc": 0.0 + }, + "4": { + "rank_median": 66.0, + "rank_mean": 1469.0, + "prob_mean": 0.018573055074975288, + "entropy_mean": 5.101497465372086, + "top1_acc": 0.0 + }, + "5": { + "rank_median": 99.0, + "rank_mean": 1175.55, + "prob_mean": 0.021596541565318716, + "entropy_mean": 4.484807601571083, + "top1_acc": 0.05 + }, + "6": { + "rank_median": 83.0, + "rank_mean": 851.5, + "prob_mean": 0.029523133340821062, + "entropy_mean": 3.9798384845256805, + "top1_acc": 0.0 + }, + "7": { + "rank_median": 72.0, + "rank_mean": 886.3, + "prob_mean": 0.03287024882073482, + "entropy_mean": 3.5318375647068025, + "top1_acc": 0.05 + }, + "8": { + "rank_median": 65.5, + "rank_mean": 990.0, + "prob_mean": 0.0446454816647341, + "entropy_mean": 3.173729035258293, + "top1_acc": 0.0 + }, + "9": { + "rank_median": 42.5, + "rank_mean": 841.75, + "prob_mean": 0.06293884984675913, + "entropy_mean": 2.6401389837265015, + "top1_acc": 0.1 + }, + "10": { + "rank_median": 61.5, + "rank_mean": 987.15, + "prob_mean": 0.09556773399166715, + "entropy_mean": 2.3502773948013784, + "top1_acc": 0.1 + }, + "11": { + "rank_median": 75.5, + "rank_mean": 936.1, + "prob_mean": 0.10289815487957425, + "entropy_mean": 2.1287053860723972, + "top1_acc": 0.1 + }, + "12": { + "rank_median": 56.0, + "rank_mean": 537.75, + "prob_mean": 0.1102860036525557, + "entropy_mean": 1.8507884941995143, + "top1_acc": 0.1 + }, + "13": { + "rank_median": 45.0, + "rank_mean": 353.45, + "prob_mean": 0.11194868451912496, + "entropy_mean": 1.6721342958509922, + "top1_acc": 0.1 + }, + "14": { + "rank_median": 28.0, + "rank_mean": 218.25, + "prob_mean": 0.08298045136286061, + "entropy_mean": 1.3356505129486322, + "top1_acc": 0.05 + }, + "15": { + "rank_median": 21.0, + "rank_mean": 152.7, + "prob_mean": 0.09413800044087912, + "entropy_mean": 1.2790355380624532, + "top1_acc": 0.05 + }, + "16": { + "rank_median": 25.5, + "rank_mean": 156.3, + "prob_mean": 0.14374329151086976, + "entropy_mean": 1.2230370048433543, + "top1_acc": 0.15 + }, + "17": { + "rank_median": 19.0, + "rank_mean": 108.4, + "prob_mean": 0.16959481575697463, + "entropy_mean": 0.982329808594659, + "top1_acc": 0.2 + }, + "18": { + "rank_median": 12.5, + "rank_mean": 69.95, + "prob_mean": 0.19127775111421444, + "entropy_mean": 0.7672953987181245, + "top1_acc": 0.2 + }, + "19": { + "rank_median": 7.5, + "rank_mean": 64.7, + "prob_mean": 0.27062316331389213, + "entropy_mean": 0.7049623723900382, + "top1_acc": 0.25 + }, + "20": { + "rank_median": 5.5, + "rank_mean": 41.3, + "prob_mean": 0.30554919984364703, + "entropy_mean": 0.5747898182224344, + "top1_acc": 0.3 + }, + "21": { + "rank_median": 4.5, + "rank_mean": 39.85, + "prob_mean": 0.33026042910579073, + "entropy_mean": 0.4324229697799177, + "top1_acc": 0.35 + }, + "22": { + "rank_median": 4.0, + "rank_mean": 26.55, + "prob_mean": 0.35049705050121727, + "entropy_mean": 0.4803217047677208, + "top1_acc": 0.35 + }, + "23": { + "rank_median": 2.5, + "rank_mean": 18.25, + "prob_mean": 0.3455437330581026, + "entropy_mean": 0.5304397597391521, + "top1_acc": 0.3 + }, + "24": { + "rank_median": 2.0, + "rank_mean": 13.7, + "prob_mean": 0.3823399636448174, + "entropy_mean": 0.32751835941043506, + "top1_acc": 0.4 + }, + "25": { + "rank_median": 1.0, + "rank_mean": 13.1, + "prob_mean": 0.4461008007887922, + "entropy_mean": 0.3247403331682063, + "top1_acc": 0.45 + }, + "26": { + "rank_median": 1.0, + "rank_mean": 16.25, + "prob_mean": 0.4586245931280814, + "entropy_mean": 0.3133909805019314, + "top1_acc": 0.45 + }, + "27": { + "rank_median": 0.5, + "rank_mean": 11.85, + "prob_mean": 0.4769530519190167, + "entropy_mean": 0.3853615205515963, + "top1_acc": 0.5 + }, + "28": { + "rank_median": 0.5, + "rank_mean": 9.65, + "prob_mean": 0.5166655593366426, + "entropy_mean": 0.24244572687020494, + "top1_acc": 0.5 + }, + "29": { + "rank_median": 0.5, + "rank_mean": 6.95, + "prob_mean": 0.5137617424570818, + "entropy_mean": 0.21511480996259963, + "top1_acc": 0.5 + }, + "30": { + "rank_median": 0.0, + "rank_mean": 5.15, + "prob_mean": 0.6711249749742791, + "entropy_mean": 0.2018442510888458, + "top1_acc": 0.7 + }, + "31": { + "rank_median": 0.0, + "rank_mean": 3.35, + "prob_mean": 0.6745306688925973, + "entropy_mean": 0.15133235824142555, + "top1_acc": 0.7 + }, + "32": { + "rank_median": 0.0, + "rank_mean": 2.25, + "prob_mean": 0.6442970273105088, + "entropy_mean": 0.2228813430716277, + "top1_acc": 0.6 + }, + "33": { + "rank_median": 0.0, + "rank_mean": 1.1, + "prob_mean": 0.7007993598404261, + "entropy_mean": 0.14786822918077186, + "top1_acc": 0.7 + }, + "34": { + "rank_median": 0.0, + "rank_mean": 0.4, + "prob_mean": 0.8490049915564896, + "entropy_mean": 0.18351739740651665, + "top1_acc": 0.85 + }, + "35": { + "rank_median": 0.0, + "rank_mean": 0.0, + "prob_mean": 0.935450142621994, + "entropy_mean": 0.17342249458097375, + "top1_acc": 1.0 + } + } +} \ No newline at end of file diff --git a/data/mi5_logit_lens/gpt2-medium.json b/data/mi5_logit_lens/gpt2-medium.json new file mode 100644 index 0000000000000000000000000000000000000000..18cbb83f90880eaaa75395e6eaeb0cc92d717bef --- /dev/null +++ b/data/mi5_logit_lens/gpt2-medium.json @@ -0,0 +1,175 @@ +{ + "model": "gpt2-medium", + "N": 24, + "L_crit": 23, + "summary": { + "0": { + "rank_median": 3800.5, + "rank_mean": 7728.6, + "prob_mean": 0.00010186947642201643, + "entropy_mean": 0.9721335422247648, + "top1_acc": 0.0 + }, + "1": { + "rank_median": 1984.5, + "rank_mean": 6044.65, + "prob_mean": 0.0022011904130948487, + "entropy_mean": 0.7699365347623826, + "top1_acc": 0.0 + }, + "2": { + "rank_median": 828.5, + "rank_mean": 4498.8, + "prob_mean": 0.04109109647653993, + "entropy_mean": 0.6389085412025451, + "top1_acc": 0.05 + }, + "3": { + "rank_median": 541.0, + "rank_mean": 3041.6, + "prob_mean": 0.030016651240949425, + "entropy_mean": 0.6866200581673183, + "top1_acc": 0.05 + }, + "4": { + "rank_median": 363.0, + "rank_mean": 2149.9, + "prob_mean": 0.001068291258318678, + "entropy_mean": 0.40619218911160715, + "top1_acc": 0.0 + }, + "5": { + "rank_median": 339.5, + "rank_mean": 2311.2, + "prob_mean": 0.004068407847321707, + "entropy_mean": 0.39103218153468333, + "top1_acc": 0.0 + }, + "6": { + "rank_median": 566.5, + "rank_mean": 1753.95, + "prob_mean": 0.024462497441430794, + "entropy_mean": 0.32791976548951424, + "top1_acc": 0.0 + }, + "7": { + "rank_median": 437.5, + "rank_mean": 1568.95, + "prob_mean": 0.035196514892251105, + "entropy_mean": 0.3463717555201896, + "top1_acc": 0.05 + }, + "8": { + "rank_median": 317.5, + "rank_mean": 1245.45, + "prob_mean": 0.058208448282272326, + "entropy_mean": 0.27896147047510167, + "top1_acc": 0.1 + }, + "9": { + "rank_median": 233.5, + "rank_mean": 960.7, + "prob_mean": 0.07738164838386005, + "entropy_mean": 0.4935258791584468, + "top1_acc": 0.1 + }, + "10": { + "rank_median": 219.0, + "rank_mean": 808.4, + "prob_mean": 0.09997233245430848, + "entropy_mean": 0.27660850273068716, + "top1_acc": 0.1 + }, + "11": { + "rank_median": 222.5, + "rank_mean": 317.45, + "prob_mean": 0.09669922734786437, + "entropy_mean": 0.2796489033149995, + "top1_acc": 0.1 + }, + "12": { + "rank_median": 96.5, + "rank_mean": 202.9, + "prob_mean": 0.05337917684983223, + "entropy_mean": 0.23089485957794348, + "top1_acc": 0.05 + }, + "13": { + "rank_median": 49.0, + "rank_mean": 120.25, + "prob_mean": 0.1527475422701456, + "entropy_mean": 0.38849720523212455, + "top1_acc": 0.15 + }, + "14": { + "rank_median": 22.5, + "rank_mean": 50.7, + "prob_mean": 0.2000067774599413, + "entropy_mean": 0.21463840760292968, + "top1_acc": 0.2 + }, + "15": { + "rank_median": 7.5, + "rank_mean": 23.6, + "prob_mean": 0.2849591718010477, + "entropy_mean": 0.21568532616719702, + "top1_acc": 0.3 + }, + "16": { + "rank_median": 2.0, + "rank_mean": 10.95, + "prob_mean": 0.3425771350360892, + "entropy_mean": 0.08329013162306871, + "top1_acc": 0.35 + }, + "17": { + "rank_median": 1.0, + "rank_mean": 6.35, + "prob_mean": 0.4389117737150439, + "entropy_mean": 0.18061973233191747, + "top1_acc": 0.45 + }, + "18": { + "rank_median": 0.0, + "rank_mean": 6.25, + "prob_mean": 0.5327236878720103, + "entropy_mean": 0.1263708216377908, + "top1_acc": 0.55 + }, + "19": { + "rank_median": 0.0, + "rank_mean": 4.15, + "prob_mean": 0.7166928410530615, + "entropy_mean": 0.06225250520121232, + "top1_acc": 0.7 + }, + "20": { + "rank_median": 0.0, + "rank_mean": 2.45, + "prob_mean": 0.7173736952117906, + "entropy_mean": 0.09497310517467454, + "top1_acc": 0.75 + }, + "21": { + "rank_median": 0.0, + "rank_mean": 0.55, + "prob_mean": 0.8408810332417488, + "entropy_mean": 0.10924532842962117, + "top1_acc": 0.85 + }, + "22": { + "rank_median": 0.0, + "rank_mean": 0.4, + "prob_mean": 0.8441994148330746, + "entropy_mean": 0.021580312155491888, + "top1_acc": 0.85 + }, + "23": { + "rank_median": 0.0, + "rank_mean": 0.0, + "prob_mean": 0.9833208918571472, + "entropy_mean": 0.033787667114919526, + "top1_acc": 1.0 + } + } +} \ No newline at end of file diff --git a/data/mi5_logit_lens/gpt2.json b/data/mi5_logit_lens/gpt2.json new file mode 100644 index 0000000000000000000000000000000000000000..f8f0abbe8ab88db49debeb703c77dee768bc48e0 --- /dev/null +++ b/data/mi5_logit_lens/gpt2.json @@ -0,0 +1,91 @@ +{ + "model": "gpt2", + "N": 12, + "L_crit": null, + "summary": { + "0": { + "rank_median": 1410.0, + "rank_mean": 7538.65, + "prob_mean": 2.0723215761599164e-06, + "entropy_mean": 1.2335072463989492, + "top1_acc": 0.0 + }, + "1": { + "rank_median": 1176.5, + "rank_mean": 5828.4, + "prob_mean": 3.7778498767853433e-06, + "entropy_mean": 0.9767906448571011, + "top1_acc": 0.0 + }, + "2": { + "rank_median": 457.5, + "rank_mean": 2046.25, + "prob_mean": 0.00020276291264603656, + "entropy_mean": 0.8634470824152232, + "top1_acc": 0.0 + }, + "3": { + "rank_median": 164.0, + "rank_mean": 735.9, + "prob_mean": 0.004644197407801644, + "entropy_mean": 0.7560844500549138, + "top1_acc": 0.0 + }, + "4": { + "rank_median": 105.0, + "rank_mean": 682.5, + "prob_mean": 0.013926062619341423, + "entropy_mean": 0.7917257328517735, + "top1_acc": 0.0 + }, + "5": { + "rank_median": 53.0, + "rank_mean": 224.05, + "prob_mean": 0.03508824152485183, + "entropy_mean": 0.5140653299167752, + "top1_acc": 0.0 + }, + "6": { + "rank_median": 19.0, + "rank_mean": 75.4, + "prob_mean": 0.09848060719183697, + "entropy_mean": 0.45440693941454813, + "top1_acc": 0.1 + }, + "7": { + "rank_median": 7.0, + "rank_mean": 39.85, + "prob_mean": 0.10439612692250629, + "entropy_mean": 0.32832070246007816, + "top1_acc": 0.1 + }, + "8": { + "rank_median": 2.0, + "rank_mean": 10.7, + "prob_mean": 0.2261893954086814, + "entropy_mean": 0.29561946169138614, + "top1_acc": 0.2 + }, + "9": { + "rank_median": 0.0, + "rank_mean": 1.9, + "prob_mean": 0.5549969809796357, + "entropy_mean": 0.30446562467295457, + "top1_acc": 0.55 + }, + "10": { + "rank_median": 0.0, + "rank_mean": 0.65, + "prob_mean": 0.720704834095075, + "entropy_mean": 0.15716689909388326, + "top1_acc": 0.75 + }, + "11": { + "rank_median": 0.0, + "rank_mean": 0.0, + "prob_mean": 0.9215896189212799, + "entropy_mean": 0.21304331419991565, + "top1_acc": 1.0 + } + } +} \ No newline at end of file diff --git a/data/normal_form/normal_form_results.json b/data/normal_form/normal_form_results.json new file mode 100644 index 0000000000000000000000000000000000000000..88d7281b98cfad83e44f18e14e9c9811fd7abf19 --- /dev/null +++ b/data/normal_form/normal_form_results.json @@ -0,0 +1,100 @@ +[ + { + "model": "BlinkDL/rwkv-4-world-3b", + "N": 32, + "L_crit": 31, + "sigmoid_k": 0.3788, + "sigmoid_lstar": 23.8166, + "sigmoid_R2": 0.9617, + "beta": null, + "beta_R2": null, + "hopf": false, + "n_sign_changes": 0, + "hopf_amplitude": 0.0, + "classification": "N/A" + }, + { + "model": "EleutherAI/pythia-160m", + "N": 12, + "L_crit": 4, + "sigmoid_k": 17.8682, + "sigmoid_lstar": 5.5044, + "sigmoid_R2": 0.4193, + "beta": 0.05, + "beta_R2": 0.0312, + "hopf": true, + "n_sign_changes": 3, + "hopf_amplitude": 1.6397, + "classification": "FAST(beta<0.35)" + }, + { + "model": "EleutherAI/pythia-1b", + "N": 16, + "L_crit": 15, + "sigmoid_k": 0.3949, + "sigmoid_lstar": 12.8744, + "sigmoid_R2": 0.9836, + "beta": null, + "beta_R2": null, + "hopf": false, + "n_sign_changes": 0, + "hopf_amplitude": 0.0, + "classification": "N/A" + }, + { + "model": "EleutherAI/pythia-70m", + "N": 6, + "L_crit": 4, + "sigmoid_k": 19.1096, + "sigmoid_lstar": 3.534, + "sigmoid_R2": 0.76, + "beta": null, + "beta_R2": null, + "hopf": false, + "n_sign_changes": 0, + "hopf_amplitude": 0.0, + "classification": "N/A" + }, + { + "model": "gpt2-large", + "N": 36, + "L_crit": 33, + "sigmoid_k": 0.2319, + "sigmoid_lstar": 27.3323, + "sigmoid_R2": 0.9805, + "beta": 0.05, + "beta_R2": 0.9991, + "hopf": false, + "n_sign_changes": 0, + "hopf_amplitude": 0.0, + "classification": "FAST(beta<0.35)" + }, + { + "model": "gpt2-medium", + "N": 24, + "L_crit": 23, + "sigmoid_k": 0.2638, + "sigmoid_lstar": 23.5723, + "sigmoid_R2": 0.8637, + "beta": null, + "beta_R2": null, + "hopf": false, + "n_sign_changes": 0, + "hopf_amplitude": 0.0, + "classification": "N/A" + }, + { + "model": "gpt2-xl", + "N": 48, + "L_crit": 43, + "sigmoid_k": 0.1194, + "sigmoid_lstar": 33.0326, + "sigmoid_R2": 0.9778, + "beta": 0.0604, + "beta_R2": 0.9804, + "hopf": false, + "n_sign_changes": 2, + "hopf_amplitude": 0.0103, + "classification": "FAST(beta<0.35)" + } +] \ No newline at end of file diff --git a/data/phase_diagram/phase_diagram.csv b/data/phase_diagram/phase_diagram.csv new file mode 100644 index 0000000000000000000000000000000000000000..3623ecad5da2c96345f8ffe2777a1c694f4d609c --- /dev/null +++ b/data/phase_diagram/phase_diagram.csv @@ -0,0 +1,4 @@ +model,g_mongo,g_random,delta_g,R2_m,R2_r,class,arch +mistralai--Mistral-7B-v0.1,1.060750,0.829601,0.231149,0.9987,0.9969,SWA_amplifies,SWA+RoPE +google--gemma-2-9b-it,0.627646,1.134796,-0.507150,0.9773,0.9765,local_in_text,SWA+RoPE(alt) +meta-llama--Llama-2-7b-hf,0.287057,0.826624,-0.539567,0.8149,0.9936,local_in_text,RoPE diff --git a/data/phase_diagram/phase_diagram.json b/data/phase_diagram/phase_diagram.json new file mode 100644 index 0000000000000000000000000000000000000000..d67a3a872e3fbdc9d447745c4088cb9374066d68 --- /dev/null +++ b/data/phase_diagram/phase_diagram.json @@ -0,0 +1,35 @@ +[ + { + "model": "mistralai--Mistral-7B-v0.1", + "g_mongo": 1.060750419523944, + "g_random": 0.8296009929924347, + "delta_g": 0.23114942653150938, + "R2_m": 0.99869, + "R2_r": 0.996923, + "arch": "SWA+RoPE", + "class": "SWA_amplifies", + "theta": 10000 + }, + { + "model": "google--gemma-2-9b-it", + "g_mongo": 0.6276459084140061, + "g_random": 1.1347958464287666, + "delta_g": -0.5071499380147605, + "R2_m": 0.977314, + "R2_r": 0.976472, + "arch": "SWA+RoPE(alt)", + "class": "local_in_text", + "theta": 10000 + }, + { + "model": "meta-llama--Llama-2-7b-hf", + "g_mongo": 0.2870574377368437, + "g_random": 0.8266242679750889, + "delta_g": -0.5395668302382453, + "R2_m": 0.814928, + "R2_r": 0.993628, + "arch": "RoPE", + "class": "local_in_text", + "theta": 10000 + } +] \ No newline at end of file diff --git a/data/q6_hierarchy/q6_lerch_hierarchy.json b/data/q6_hierarchy/q6_lerch_hierarchy.json new file mode 100644 index 0000000000000000000000000000000000000000..604f5e2b648da48dee11c22dc454e77a217c60f6 --- /dev/null +++ b/data/q6_hierarchy/q6_lerch_hierarchy.json @@ -0,0 +1,58 @@ +{ + "experiment": "Q6_lerch_hierarchy", + "date": "2026-04-23", + "claim_tested": "err_Pade(z) = z^2/(2+3z+z^2) predicts observed error before measuring gamma_obs", + "result": "NEGATIVE \u2014 mean ratio err_obs/err_pred = 88.6x for standard models", + "valid_regime": "z < 0.01 (theta > 290K): err_pred < 0.001, negligible", + "invalid_regime": "z > 0.03: model-specific spread 5-1300x larger than prediction", + "math_hierarchy": { + "level_0": { + "formula": "gamma=1", + "error": "O(z)" + }, + "level_1": { + "formula": "gamma=1-z", + "error": "O(z^2)" + }, + "pade": { + "formula": "(2-z)/(2+z)", + "error": "O(z^2/(2+3z))" + }, + "lerch": { + "formula": "1/(1+z)", + "error": "O(z^3/(1+z)^2)", + "improvement": "~5%_globally" + }, + "exact": { + "formula": "Lerch_sum", + "error": "0 (pre-softmax)" + } + }, + "paper_claim_revised": "Hierarchy valid mathematically. z predicts REGIME not exact error. For non-anomalous large-theta models (z<0.01): any formula works. For z~0.3 (theta=10K): use 1/(1+z). Anomalous models (GQA/SWA/Hagedorn): must measure.", + "per_model": { + "pythia-70m": { + "z": 0.2896, + "err_pred": 0.0284, + "err_obs": 0.0056, + "ratio": 0.2 + }, + "Qwen2.5-7B": { + "z": 0.0029, + "err_pred": 0.0, + "err_obs": 0.0001, + "ratio": 26.8 + }, + "LLaMA-2-7b": { + "z": 0.2896, + "err_pred": 0.0284, + "err_obs": 0.4884, + "ratio": 17.2 + }, + "SmolLM2-135M": { + "z": 0.029, + "err_pred": 0.0004, + "err_obs": 0.2239, + "ratio": 557.1 + } + } +} \ No newline at end of file diff --git a/data/rel12_passkey/results.json b/data/rel12_passkey/results.json new file mode 100644 index 0000000000000000000000000000000000000000..5799b8b77ba30c06c3be257c6b08d2e76e9ce727 --- /dev/null +++ b/data/rel12_passkey/results.json @@ -0,0 +1,56 @@ +[ + { + "d": 100, + "f_dead": 0.3005, + "T_cross": 628.3, + "pred": "PASS", + "match_rate": 0.0, + "n_trials": 60, + "confirms_rel12": false + }, + { + "d": 300, + "f_dead": 0.4197, + "T_cross": 628.3, + "pred": "PASS", + "match_rate": 0.0, + "n_trials": 60, + "confirms_rel12": false + }, + { + "d": 500, + "f_dead": 0.4752, + "T_cross": 628.3, + "pred": "PASS", + "match_rate": 0.0, + "n_trials": 60, + "confirms_rel12": false + }, + { + "d": 700, + "f_dead": 0.5117, + "T_cross": 628.3, + "pred": "FAIL", + "match_rate": 0.0, + "n_trials": 60, + "confirms_rel12": true + }, + { + "d": 1000, + "f_dead": 0.5505, + "T_cross": 628.3, + "pred": "FAIL", + "match_rate": 0.0, + "n_trials": 60, + "confirms_rel12": true + }, + { + "d": 1500, + "f_dead": 0.5945, + "T_cross": 628.3, + "pred": "FAIL", + "match_rate": 0.0, + "n_trials": 60, + "confirms_rel12": true + } +] \ No newline at end of file diff --git a/data/rlhf_gamma/dolly-pythia-70m_mongo_rlhf.json b/data/rlhf_gamma/dolly-pythia-70m_mongo_rlhf.json new file mode 100644 index 0000000000000000000000000000000000000000..bf41614d8e55ae5b972e62834cceeee6171657d5 --- /dev/null +++ b/data/rlhf_gamma/dolly-pythia-70m_mongo_rlhf.json @@ -0,0 +1,7 @@ +{ + "pair": "dolly-pythia-70m", + "gamma_base": 0.7476017873166874, + "R2_base": 0.984269, + "gamma_instruct": null, + "status": "base_only \u2014 instruct measurement pending" +} \ No newline at end of file diff --git a/data/rlhf_gamma/pythia-70m_mongo_rlhf.json b/data/rlhf_gamma/pythia-70m_mongo_rlhf.json new file mode 100644 index 0000000000000000000000000000000000000000..5bc1fc7c4cb50e011aa6c2cd1b15c0ed65ad9c53 --- /dev/null +++ b/data/rlhf_gamma/pythia-70m_mongo_rlhf.json @@ -0,0 +1,7 @@ +{ + "pair": "pythia-70m", + "gamma_base": 0.7476017873166874, + "R2_base": 0.984269, + "gamma_instruct": null, + "status": "base_only \u2014 instruct measurement pending" +} \ No newline at end of file diff --git a/data/rlhf_gamma/rlhf_gamma_all_mongo.json b/data/rlhf_gamma/rlhf_gamma_all_mongo.json new file mode 100644 index 0000000000000000000000000000000000000000..9ed65125a526521fa8b23a5a329ca375cec06426 --- /dev/null +++ b/data/rlhf_gamma/rlhf_gamma_all_mongo.json @@ -0,0 +1,9 @@ +[ + { + "pair": "pythia-70m", + "gamma_base": 0.7476017873166874, + "R2_base": 0.984269, + "gamma_instruct": null, + "status": "base_only \u2014 instruct measurement pending" + } +] \ No newline at end of file diff --git a/data/rlhf_gamma/rlhf_real_pairs.json b/data/rlhf_gamma/rlhf_real_pairs.json new file mode 100644 index 0000000000000000000000000000000000000000..45cc879ef0929e0e8bda3e07e60099bc1d6fdbce --- /dev/null +++ b/data/rlhf_gamma/rlhf_real_pairs.json @@ -0,0 +1,37 @@ +[ + { + "pair": "smollm2-360m", + "base": "HuggingFaceTB/SmolLM2-360M", + "instruct": "HuggingFaceTB/SmolLM2-360M-Instruct", + "theta": 10000, + "gamma_pred": 1.0, + "gamma_base": 0.9192337618040853, + "R2_base": 0.9916, + "gamma_instruct": 0.9127684667598348, + "R2_instruct": 0.9938, + "delta_gamma": -0.00646529504425053, + "delta_pct": 0.7033352464732977, + "interpretation": "H3: no change \u2014 RLHF acts elsewhere, not positional attention", + "significant": false, + "attn_base": { + "10": 0.007876912066221849, + "20": 0.005768730380768829, + "50": 0.002727720131567821, + "100": 0.0014364854118707627, + "200": 0.0007443138907158442, + "500": 0.00028645839140396313, + "1000": 0.0001556541892953689, + "2000": 6.468147635719013e-05 + }, + "attn_instruct": { + "10": 0.007941762741293133, + "20": 0.00567333775366933, + "50": 0.002715980579916959, + "100": 0.0014180658569782167, + "200": 0.0007151647645909165, + "500": 0.000281991663518415, + "1000": 0.00015078636685665004, + "2000": 7.066360648636639e-05 + } + } +] \ No newline at end of file diff --git a/data/rlhf_gamma/rlhf_summary.json b/data/rlhf_gamma/rlhf_summary.json new file mode 100644 index 0000000000000000000000000000000000000000..88c1307813be6962a268a65c396f4245a32d0620 --- /dev/null +++ b/data/rlhf_gamma/rlhf_summary.json @@ -0,0 +1,35 @@ +{ + "experiments_done": "2026-04-21", + "pairs": [ + { + "pair": "smollm2-135m", + "base": "HuggingFaceTB/SmolLM2-135M", + "instruct": "HuggingFaceTB/SmolLM2-135M-Instruct", + "theta": 10000, + "gamma_base": 0.9019, + "gamma_instruct": 0.9164, + "delta_gamma": 0.0145, + "delta_pct": 1.6, + "R2_base": 0.9995, + "R2_instruct": 0.9989, + "hypothesis": "H3: no change", + "significant": false + }, + { + "pair": "qwen-0.5b", + "base": "Qwen/Qwen2.5-0.5B", + "instruct": "Qwen/Qwen2.5-0.5B-Instruct", + "theta": 1000000, + "gamma_base": 0.7992, + "gamma_instruct": 0.7169, + "delta_gamma": -0.0824, + "delta_pct": 10.3, + "R2_base": 0.9544, + "R2_instruct": 0.9567, + "hypothesis": "H2: more global attention after RLHF", + "significant": true + } + ], + "finding": "Small models (theta=10K): SFT does not change gamma significantly. Large-theta models (theta=1M): instruct version has lower gamma = more global attention. Consistent with instruction-following requiring long-range context.", + "caveat": "Only 2 pairs measured. More pairs needed for robust claim. GPU experiments pending for larger models." +} \ No newline at end of file diff --git a/data/rope_subspace/EleutherAI--pythia-70m_subspace.json b/data/rope_subspace/EleutherAI--pythia-70m_subspace.json new file mode 100644 index 0000000000000000000000000000000000000000..99f325bf3364756148fe92d310f98abcb0d913f9 --- /dev/null +++ b/data/rope_subspace/EleutherAI--pythia-70m_subspace.json @@ -0,0 +1,636 @@ +{ + "model": "EleutherAI/pythia-70m", + "theta": 10000, + "d_head": 64, + "n_layers": 6, + "n_heads": 8, + "d_long": 1000, + "n_dead_pairs": 17, + "k_pred": 35, + "k_obs_90_mean": 21.5, + "k_obs_90_std": 8.7, + "err_90_pct": 38.6, + "snr_active_vs_dead_mean": 1.03, + "layers": [ + { + "layer": 0, + "k_obs_90_mean": 26.875, + "snr_active_vs_dead": 1.1495240852329491, + "heads": [ + { + "head": 0, + "k_obs_90": 28, + "k_obs_50": 8, + "k_pred": 35, + "err_90_pct": 20.0, + "k_dead": 34, + "k_active": 30, + "var_dead": 1.0019279216066934e-05, + "var_act": 1.2584573596541304e-05, + "snr_ratio": 1.2560356956940542 + }, + { + "head": 1, + "k_obs_90": 24, + "k_obs_50": 4, + "k_pred": 35, + "err_90_pct": 31.428571428571427, + "k_dead": 34, + "k_active": 30, + "var_dead": 4.331851960159838e-05, + "var_act": 4.879820335190743e-05, + "snr_ratio": 1.1264974582282217 + }, + { + "head": 2, + "k_obs_90": 26, + "k_obs_50": 8, + "k_pred": 35, + "err_90_pct": 25.71428571428571, + "k_dead": 34, + "k_active": 30, + "var_dead": 2.3921231331769377e-05, + "var_act": 3.0288869311334565e-05, + "snr_ratio": 1.2661918454388505 + }, + { + "head": 3, + "k_obs_90": 23, + "k_obs_50": 4, + "k_pred": 35, + "err_90_pct": 34.285714285714285, + "k_dead": 34, + "k_active": 30, + "var_dead": 4.9896434575202875e-06, + "var_act": 4.899123723589582e-06, + "snr_ratio": 0.9818582797429034 + }, + { + "head": 4, + "k_obs_90": 25, + "k_obs_50": 4, + "k_pred": 35, + "err_90_pct": 28.57142857142857, + "k_dead": 34, + "k_active": 30, + "var_dead": 5.57630555704236e-05, + "var_act": 5.8389643527334556e-05, + "snr_ratio": 1.047102636018415 + }, + { + "head": 5, + "k_obs_90": 29, + "k_obs_50": 9, + "k_pred": 35, + "err_90_pct": 17.142857142857142, + "k_dead": 34, + "k_active": 30, + "var_dead": 2.726745151448995e-05, + "var_act": 2.89542913378682e-05, + "snr_ratio": 1.0618627215902132 + }, + { + "head": 6, + "k_obs_90": 30, + "k_obs_50": 11, + "k_pred": 35, + "err_90_pct": 14.285714285714285, + "k_dead": 34, + "k_active": 30, + "var_dead": 3.198338163201697e-05, + "var_act": 3.691744132083841e-05, + "snr_ratio": 1.1542694450299384 + }, + { + "head": 7, + "k_obs_90": 30, + "k_obs_50": 10, + "k_pred": 35, + "err_90_pct": 14.285714285714285, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.00010517389455344528, + "var_act": 0.0001369758101645857, + "snr_ratio": 1.3023746001209962 + } + ] + }, + { + "layer": 1, + "k_obs_90_mean": 29.625, + "snr_active_vs_dead": 1.019355275928171, + "heads": [ + { + "head": 0, + "k_obs_90": 31, + "k_obs_50": 11, + "k_pred": 35, + "err_90_pct": 11.428571428571429, + "k_dead": 34, + "k_active": 30, + "var_dead": 2.5180415832437575e-05, + "var_act": 2.4832079361658543e-05, + "snr_ratio": 0.9861663342153137 + }, + { + "head": 1, + "k_obs_90": 28, + "k_obs_50": 8, + "k_pred": 35, + "err_90_pct": 20.0, + "k_dead": 34, + "k_active": 30, + "var_dead": 1.9814948245766573e-05, + "var_act": 1.8872216969612055e-05, + "snr_ratio": 0.9524231798698182 + }, + { + "head": 2, + "k_obs_90": 31, + "k_obs_50": 10, + "k_pred": 35, + "err_90_pct": 11.428571428571429, + "k_dead": 34, + "k_active": 30, + "var_dead": 1.0731777365435846e-05, + "var_act": 8.605376024206635e-06, + "snr_ratio": 0.8018592754321336 + }, + { + "head": 3, + "k_obs_90": 29, + "k_obs_50": 9, + "k_pred": 35, + "err_90_pct": 17.142857142857142, + "k_dead": 34, + "k_active": 30, + "var_dead": 3.0166356737026945e-05, + "var_act": 2.9634320526383817e-05, + "snr_ratio": 0.9823632267680068 + }, + { + "head": 4, + "k_obs_90": 28, + "k_obs_50": 8, + "k_pred": 35, + "err_90_pct": 20.0, + "k_dead": 34, + "k_active": 30, + "var_dead": 6.330292671918869e-05, + "var_act": 7.00203818269074e-05, + "snr_ratio": 1.1061160099500813 + }, + { + "head": 5, + "k_obs_90": 30, + "k_obs_50": 10, + "k_pred": 35, + "err_90_pct": 14.285714285714285, + "k_dead": 34, + "k_active": 30, + "var_dead": 2.081714410451241e-05, + "var_act": 1.9878509192494676e-05, + "snr_ratio": 0.9549104400576873 + }, + { + "head": 6, + "k_obs_90": 31, + "k_obs_50": 11, + "k_pred": 35, + "err_90_pct": 11.428571428571429, + "k_dead": 34, + "k_active": 30, + "var_dead": 4.048247865284793e-05, + "var_act": 4.738761344924569e-05, + "snr_ratio": 1.1705709199538123 + }, + { + "head": 7, + "k_obs_90": 29, + "k_obs_50": 9, + "k_pred": 35, + "err_90_pct": 17.142857142857142, + "k_dead": 34, + "k_active": 30, + "var_dead": 9.403538570040837e-05, + "var_act": 0.00011288316454738379, + "snr_ratio": 1.2004328211785147 + } + ] + }, + { + "layer": 2, + "k_obs_90_mean": 28.5, + "snr_active_vs_dead": 0.9347101243340801, + "heads": [ + { + "head": 0, + "k_obs_90": 29, + "k_obs_50": 9, + "k_pred": 35, + "err_90_pct": 17.142857142857142, + "k_dead": 34, + "k_active": 30, + "var_dead": 6.118758756201714e-05, + "var_act": 5.541780774365179e-05, + "snr_ratio": 0.9057034121794595 + }, + { + "head": 1, + "k_obs_90": 28, + "k_obs_50": 9, + "k_pred": 35, + "err_90_pct": 20.0, + "k_dead": 34, + "k_active": 30, + "var_dead": 6.667244451818988e-05, + "var_act": 6.633422162849456e-05, + "snr_ratio": 0.9949270813892219 + }, + { + "head": 2, + "k_obs_90": 30, + "k_obs_50": 10, + "k_pred": 35, + "err_90_pct": 14.285714285714285, + "k_dead": 34, + "k_active": 30, + "var_dead": 4.0906830690801144e-05, + "var_act": 3.498488149489276e-05, + "snr_ratio": 0.8552332226394332 + }, + { + "head": 3, + "k_obs_90": 30, + "k_obs_50": 10, + "k_pred": 35, + "err_90_pct": 14.285714285714285, + "k_dead": 34, + "k_active": 30, + "var_dead": 9.686067642178386e-06, + "var_act": 9.512852557236329e-06, + "snr_ratio": 0.982116987671574 + }, + { + "head": 4, + "k_obs_90": 25, + "k_obs_50": 5, + "k_pred": 35, + "err_90_pct": 28.57142857142857, + "k_dead": 34, + "k_active": 30, + "var_dead": 2.511212369427085e-05, + "var_act": 2.4494846002198756e-05, + "snr_ratio": 0.9754190973648311 + }, + { + "head": 5, + "k_obs_90": 30, + "k_obs_50": 9, + "k_pred": 35, + "err_90_pct": 14.285714285714285, + "k_dead": 34, + "k_active": 30, + "var_dead": 3.630656283348799e-05, + "var_act": 2.8901031328132376e-05, + "snr_ratio": 0.7960277227192458 + }, + { + "head": 6, + "k_obs_90": 30, + "k_obs_50": 10, + "k_pred": 35, + "err_90_pct": 14.285714285714285, + "k_dead": 34, + "k_active": 30, + "var_dead": 1.3595455129689071e-05, + "var_act": 1.242047619598452e-05, + "snr_ratio": 0.9135755415267981 + }, + { + "head": 7, + "k_obs_90": 26, + "k_obs_50": 7, + "k_pred": 35, + "err_90_pct": 25.71428571428571, + "k_dead": 34, + "k_active": 30, + "var_dead": 3.240483420086093e-05, + "var_act": 3.417666448513046e-05, + "snr_ratio": 1.0546779291820763 + } + ] + }, + { + "layer": 3, + "k_obs_90_mean": 17.75, + "snr_active_vs_dead": 0.9830222722045272, + "heads": [ + { + "head": 0, + "k_obs_90": 20, + "k_obs_50": 4, + "k_pred": 35, + "err_90_pct": 42.857142857142854, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.00043395993998274207, + "var_act": 0.0005869995220564306, + "snr_ratio": 1.352658313869055 + }, + { + "head": 1, + "k_obs_90": 11, + "k_obs_50": 1, + "k_pred": 35, + "err_90_pct": 68.57142857142857, + "k_dead": 34, + "k_active": 30, + "var_dead": 1.767167850630358e-05, + "var_act": 1.2928276191814803e-05, + "snr_ratio": 0.7315816353054174 + }, + { + "head": 2, + "k_obs_90": 20, + "k_obs_50": 4, + "k_pred": 35, + "err_90_pct": 42.857142857142854, + "k_dead": 34, + "k_active": 30, + "var_dead": 1.2255043657205533e-05, + "var_act": 8.937295206123963e-06, + "snr_ratio": 0.7292747971235788 + }, + { + "head": 3, + "k_obs_90": 20, + "k_obs_50": 6, + "k_pred": 35, + "err_90_pct": 42.857142857142854, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0005289488472044468, + "var_act": 0.0007070901338011026, + "snr_ratio": 1.336783577847591 + }, + { + "head": 4, + "k_obs_90": 10, + "k_obs_50": 1, + "k_pred": 35, + "err_90_pct": 71.42857142857143, + "k_dead": 34, + "k_active": 30, + "var_dead": 1.8021095456788316e-05, + "var_act": 1.073166095011402e-05, + "snr_ratio": 0.5955054386311517 + }, + { + "head": 5, + "k_obs_90": 24, + "k_obs_50": 5, + "k_pred": 35, + "err_90_pct": 31.428571428571427, + "k_dead": 34, + "k_active": 30, + "var_dead": 2.387595668551512e-05, + "var_act": 2.0395393221406266e-05, + "snr_ratio": 0.8542230426961924 + }, + { + "head": 6, + "k_obs_90": 27, + "k_obs_50": 8, + "k_pred": 35, + "err_90_pct": 22.857142857142858, + "k_dead": 34, + "k_active": 30, + "var_dead": 1.6957081243162975e-05, + "var_act": 1.393092406942742e-05, + "snr_ratio": 0.8215401606042402 + }, + { + "head": 7, + "k_obs_90": 10, + "k_obs_50": 1, + "k_pred": 35, + "err_90_pct": 71.42857142857143, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0027891683857887983, + "var_act": 0.004023685585707426, + "snr_ratio": 1.4426112115589915 + } + ] + }, + { + "layer": 4, + "k_obs_90_mean": 11.5, + "snr_active_vs_dead": 1.179930353539545, + "heads": [ + { + "head": 0, + "k_obs_90": 17, + "k_obs_50": 1, + "k_pred": 35, + "err_90_pct": 51.42857142857142, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0003967596567235887, + "var_act": 0.0007541452068835497, + "snr_ratio": 1.9007608062030878 + }, + { + "head": 1, + "k_obs_90": 1, + "k_obs_50": 1, + "k_pred": 35, + "err_90_pct": 97.14285714285714, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0011889338493347168, + "var_act": 0.0007640656549483538, + "snr_ratio": 0.642647742541142 + }, + { + "head": 2, + "k_obs_90": 15, + "k_obs_50": 4, + "k_pred": 35, + "err_90_pct": 57.14285714285714, + "k_dead": 34, + "k_active": 30, + "var_dead": 1.9490307749947533e-05, + "var_act": 7.792907126713544e-06, + "snr_ratio": 0.39983497576632915 + }, + { + "head": 3, + "k_obs_90": 14, + "k_obs_50": 1, + "k_pred": 35, + "err_90_pct": 60.0, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0008710233960300684, + "var_act": 0.0010841175680980086, + "snr_ratio": 1.2446480448108836 + }, + { + "head": 4, + "k_obs_90": 12, + "k_obs_50": 2, + "k_pred": 35, + "err_90_pct": 65.71428571428571, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0015370206674560905, + "var_act": 0.0012197827454656363, + "snr_ratio": 0.7936020448514112 + }, + { + "head": 5, + "k_obs_90": 13, + "k_obs_50": 3, + "k_pred": 35, + "err_90_pct": 62.857142857142854, + "k_dead": 34, + "k_active": 30, + "var_dead": 2.0562669305945747e-05, + "var_act": 3.736602138815215e-06, + "snr_ratio": 0.1817177479004158 + }, + { + "head": 6, + "k_obs_90": 16, + "k_obs_50": 1, + "k_pred": 35, + "err_90_pct": 54.285714285714285, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0025106726679950953, + "var_act": 0.0020829441491514444, + "snr_ratio": 0.8296358879730624 + }, + { + "head": 7, + "k_obs_90": 4, + "k_obs_50": 1, + "k_pred": 35, + "err_90_pct": 88.57142857142857, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0003777504025492817, + "var_act": 0.0013019528705626726, + "snr_ratio": 3.446595578270027 + } + ] + }, + { + "layer": 5, + "k_obs_90_mean": 14.75, + "snr_active_vs_dead": 0.9355305356560584, + "heads": [ + { + "head": 0, + "k_obs_90": 25, + "k_obs_50": 7, + "k_pred": 35, + "err_90_pct": 28.57142857142857, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0004852833808399737, + "var_act": 0.0008678752346895635, + "snr_ratio": 1.7883885316636554 + }, + { + "head": 1, + "k_obs_90": 5, + "k_obs_50": 1, + "k_pred": 35, + "err_90_pct": 85.71428571428571, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0012760712997987866, + "var_act": 0.0010433715069666505, + "snr_ratio": 0.8176435801929937 + }, + { + "head": 2, + "k_obs_90": 12, + "k_obs_50": 3, + "k_pred": 35, + "err_90_pct": 65.71428571428571, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0001333903637714684, + "var_act": 1.0775472219393123e-05, + "snr_ratio": 0.08078148851196446 + }, + { + "head": 3, + "k_obs_90": 26, + "k_obs_50": 6, + "k_pred": 35, + "err_90_pct": 25.71428571428571, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.00044718952267430723, + "var_act": 0.0006745407590642571, + "snr_ratio": 1.5084001823699524 + }, + { + "head": 4, + "k_obs_90": 4, + "k_obs_50": 1, + "k_pred": 35, + "err_90_pct": 88.57142857142857, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.001186757697723806, + "var_act": 0.0009325856808573008, + "snr_ratio": 0.7858265270662814 + }, + { + "head": 5, + "k_obs_90": 10, + "k_obs_50": 2, + "k_pred": 35, + "err_90_pct": 71.42857142857143, + "k_dead": 34, + "k_active": 30, + "var_dead": 5.551165668293834e-05, + "var_act": 5.774797045887681e-06, + "snr_ratio": 0.10402854619963152 + }, + { + "head": 6, + "k_obs_90": 25, + "k_obs_50": 6, + "k_pred": 35, + "err_90_pct": 28.57142857142857, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0007743753958493471, + "var_act": 0.0011070130858570337, + "snr_ratio": 1.429556117563999 + }, + { + "head": 7, + "k_obs_90": 11, + "k_obs_50": 2, + "k_pred": 35, + "err_90_pct": 68.57142857142857, + "k_dead": 34, + "k_active": 30, + "var_dead": 0.0011292260605841875, + "var_act": 0.0010949193965643644, + "snr_ratio": 0.9696193116799887 + } + ] + } + ], + "interpretation": "k_pred overestimates observed subspace rank by 38%. SNR=1.03x: active dims only marginally more informative than dead dims in QK weights. Suggests pos/sem separation in weights is not as clean as pure theory predicts.", + "implication_for_paper": "Limitation for \u00a710: weight-space projection does not cleanly separate into active/dead subspaces. The attention output (A(d)) follows power law, but the QK weight structure does not perfectly encode it." +} \ No newline at end of file diff --git a/data/rope_subspace/HuggingFaceTB--SmolLM2-135M_subspace.json b/data/rope_subspace/HuggingFaceTB--SmolLM2-135M_subspace.json new file mode 100644 index 0000000000000000000000000000000000000000..c179b15a1d8e17a58ad23dfe643b92be3caed7aa --- /dev/null +++ b/data/rope_subspace/HuggingFaceTB--SmolLM2-135M_subspace.json @@ -0,0 +1,15 @@ +{ + "model": "HuggingFaceTB/SmolLM2-135M", + "theta": 100000, + "d_head": 64, + "n_layers": 30, + "n_heads": 9, + "d_long": 1000, + "n_dead_pairs": 14, + "k_pred": 28, + "k_obs_90_mean": null, + "k_obs_90_std": null, + "err_90_pct": null, + "snr_active_vs_dead_mean": null, + "layers": [] +} \ No newline at end of file diff --git a/data/rope_subspace/Qwen--Qwen2.5-0.5B_subspace.json b/data/rope_subspace/Qwen--Qwen2.5-0.5B_subspace.json new file mode 100644 index 0000000000000000000000000000000000000000..fa60b46bdca7d7eb0c71b394ed7af33ed15bf35a --- /dev/null +++ b/data/rope_subspace/Qwen--Qwen2.5-0.5B_subspace.json @@ -0,0 +1,15 @@ +{ + "model": "Qwen/Qwen2.5-0.5B", + "theta": 1000000.0, + "d_head": 64, + "n_layers": 24, + "n_heads": 14, + "d_long": 1000, + "n_dead_pairs": 11, + "k_pred": 23, + "k_obs_90_mean": null, + "k_obs_90_std": null, + "err_90_pct": null, + "snr_active_vs_dead_mean": null, + "layers": [] +} \ No newline at end of file diff --git a/data/temperature_sweep/pythia-70m_temp_sweep.json b/data/temperature_sweep/pythia-70m_temp_sweep.json new file mode 100644 index 0000000000000000000000000000000000000000..21c8bb4e0533272fd57360aabfa6d73c51a1adb5 --- /dev/null +++ b/data/temperature_sweep/pythia-70m_temp_sweep.json @@ -0,0 +1,151 @@ +{ + "model": "pythia-70m", + "gamma_pade": 0.7522013138014093, + "taus": [ + 0.3, + 0.5, + 0.7, + 0.9, + 1.0, + 1.3, + 1.5, + 2.0 + ], + "distances": [ + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "sweep": { + "0.3": { + "gamma_obs": 0.6748597632499864, + "r2": 0.9534977917256326, + "means": [ + 0.00877254337325408, + 0.0036872063916038576, + 0.0020279962992912773, + 0.0023177078417021516, + 0.001457432104809327, + 0.0005484562697666104, + 0.00023905000881688134, + 0.00023942190021981807 + ], + "ratio_to_baseline": null + }, + "0.5": { + "gamma_obs": 0.7227366013549389, + "r2": 0.9707959231559796, + "means": [ + 0.010344111954004297, + 0.00436339228456038, + 0.0023535067863501057, + 0.002657204855271781, + 0.0009774632758799996, + 0.0006294924672675095, + 0.0002551649518809429, + 0.00020861676636667388 + ], + "ratio_to_baseline": null + }, + "0.7": { + "gamma_obs": 0.7576206546982505, + "r2": 0.9896987380256582, + "means": [ + 0.010525427156891155, + 0.005505740524515406, + 0.0031730630406147813, + 0.0024117221681990487, + 0.0011422458647151262, + 0.0006639790516727672, + 0.0002784718984397077, + 0.00018243189808923096 + ], + "ratio_to_baseline": null + }, + "0.9": { + "gamma_obs": 0.7486571872031924, + "r2": 0.9890622233987988, + "means": [ + 0.010008190793450921, + 0.0063505818355932, + 0.0030566836857663783, + 0.0025352109068787995, + 0.0012943572933889096, + 0.0006878345474180809, + 0.00029268555346549066, + 0.00019452083441883083 + ], + "ratio_to_baseline": null + }, + "1.0": { + "gamma_obs": 0.7493589625280966, + "r2": 0.9868773001810779, + "means": [ + 0.009506619756575675, + 0.0067441158721016515, + 0.0033795992821170425, + 0.002487490379927395, + 0.0014098713588383464, + 0.0007273848762755126, + 0.0002981155557234969, + 0.00018959359564745014 + ], + "ratio_to_baseline": null + }, + "1.3": { + "gamma_obs": 0.7266446718535118, + "r2": 0.9817730246716646, + "means": [ + 0.009125704855088974, + 0.007557276725613823, + 0.003470481692218325, + 0.0025989928247225993, + 0.0016277744002258661, + 0.0008231677982621477, + 0.0003737761283648271, + 0.00019585618001340466 + ], + "ratio_to_baseline": 0.9696883712474004 + }, + "1.5": { + "gamma_obs": 0.7207624243380348, + "r2": 0.9767137669640669, + "means": [ + 0.0086186606850889, + 0.008314684006230285, + 0.003497378026885498, + 0.0026497176544378618, + 0.0016313805506797505, + 0.0008687044893223275, + 0.00038870242459678497, + 0.00019992703013384821 + ], + "ratio_to_baseline": 0.9618386652858781 + }, + "2.0": { + "gamma_obs": 0.6825743976152973, + "r2": 0.9646772943009717, + "means": [ + 0.007942009078235262, + 0.00830998922077318, + 0.004084856254566047, + 0.0028274619330962494, + 0.0018619736459287296, + 0.0010147130453131265, + 0.000489281154780959, + 0.00021840557575261404 + ], + "ratio_to_baseline": 0.9108777391712382 + } + }, + "power_law_fit": { + "gamma_0": 0.7230796296447487, + "beta": -0.008319412810300915, + "r2": 0.014400344511653172 + } +} \ No newline at end of file diff --git a/data/thermodynamics/thermodynamics_results.json b/data/thermodynamics/thermodynamics_results.json new file mode 100644 index 0000000000000000000000000000000000000000..177a940abc5e2a3e0bbf01b5d12112f4b6e5c074 --- /dev/null +++ b/data/thermodynamics/thermodynamics_results.json @@ -0,0 +1,354 @@ +[ + { + "stem": "EleutherAI--pythia-1.4b_mongo", + "model": "EleutherAI/pythia-1.4b", + "corpus": "mongo", + "gamma": 0.7050725013322717, + "log_A": -3.342801964634624, + "R2": 0.841258, + "theta": 10000, + "T_attn": 1.4182938607170854, + "Z": 1.0429471611449286, + "U": 4.940811600533233, + "S": 3.525680908079337, + "F": -0.042050514279865635, + "Cv": 2.255279078095283, + "chi": 3.3906638225234524, + "xi": 2.8616017011745782, + "D90": 1440, + "delta_H_90": -0.03530786704043906, + "efficiency": 0.8402020405624034, + "phase": "A (deconf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "EleutherAI--pythia-14m_random", + "model": "EleutherAI/pythia-14m", + "corpus": "random", + "gamma": 1.003714187534367, + "log_A": -1.6477058895472607, + "R2": 0.977698, + "theta": 10000, + "T_attn": 0.9962995566063574, + "Z": 1.6351366914147096, + "U": 3.337279243897935, + "S": 3.841400929030726, + "F": -0.4917264041664036, + "Cv": 5.776267705944429, + "chi": 269.23788601063444, + "xi": 269.73757706845754, + "D90": 836, + "delta_H_90": -0.07484146013435433, + "efficiency": NaN, + "phase": "* Hagedorn", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "EleutherAI--pythia-160m_random", + "model": "EleutherAI/pythia-160m", + "corpus": "random", + "gamma": 1.0171452847779678, + "log_A": -1.8268598516498322, + "R2": 0.981723, + "theta": 10000, + "T_attn": 0.9831437209270351, + "Z": 1.3076944502064685, + "U": 3.2604199948311714, + "S": 3.5845864491248514, + "F": -0.26826562498651896, + "Cv": 5.907602624858324, + "chi": 58.32507380017551, + "xi": 58.82365714340857, + "D90": 799, + "delta_H_90": -0.07731478366925829, + "efficiency": NaN, + "phase": "* Hagedorn", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "EleutherAI--pythia-1b_mongo", + "model": "EleutherAI/pythia-1b", + "corpus": "mongo", + "gamma": 0.9311078627189842, + "log_A": -2.350543685121484, + "R2": 0.983104, + "theta": 10000, + "T_attn": 1.0739894270464432, + "Z": 1.0474447268475386, + "U": 3.754738560977521, + "S": 3.5424202012696164, + "F": -0.04635360468928273, + "Cv": 4.966270425874297, + "chi": 14.515444569834292, + "xi": 14.009496728934652, + "D90": 1028, + "delta_H_90": -0.062104239203459546, + "efficiency": 3.596908093993407, + "phase": "A (deconf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "EleutherAI--pythia-2.8b_mongo", + "model": "EleutherAI/pythia-2.8b", + "corpus": "mongo", + "gamma": 0.6741618914822415, + "log_A": -3.179715570803609, + "R2": 0.999287, + "theta": 10000, + "T_attn": 1.483323238282361, + "Z": 1.4333150076186325, + "U": 5.077369750148241, + "S": 3.7829591430819782, + "F": -0.3599899485673239, + "Cv": 1.9534892119565124, + "chi": 3.069008731204008, + "xi": 2.5362364668886053, + "D90": 1476, + "delta_H_90": -0.033000193771156476, + "efficiency": 0.7604963315243575, + "phase": "A (deconf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "EleutherAI--pythia-31m_mongo", + "model": "EleutherAI/pythia-31m", + "corpus": "mongo", + "gamma": 1.2350013988825523, + "log_A": -0.8481173688844952, + "R2": 0.973742, + "theta": 10000, + "T_attn": 0.8097156820266074, + "Z": 1.9450370304032438, + "U": 2.125042170604823, + "S": 3.289711069038281, + "F": -0.6652810156569092, + "Cv": 6.861402218358147, + "chi": 4.255293818483924, + "xi": 4.737717523253608, + "D90": 246, + "delta_H_90": -0.11502902082432076, + "efficiency": NaN, + "phase": "B (conf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "EleutherAI--pythia-410m_mongo", + "model": "EleutherAI/pythia-410m", + "corpus": "mongo", + "gamma": 1.0218530106365162, + "log_A": -1.7669627940483377, + "R2": 0.981594, + "theta": 10000, + "T_attn": 0.9786143306238302, + "Z": 1.36735300639609, + "U": 3.233561483856463, + "S": 3.6171012961410596, + "F": -0.312876758784052, + "Cv": 5.951976455162578, + "chi": 45.760285236351216, + "xi": 46.25848377902982, + "D90": 785, + "delta_H_90": -0.07825223337361463, + "efficiency": NaN, + "phase": "* Hagedorn", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "EleutherAI--pythia-70m_mongo", + "model": "EleutherAI/pythia-70m", + "corpus": "mongo", + "gamma": 0.7476017873166874, + "log_A": -2.391511197086578, + "R2": 0.984269, + "theta": 10000, + "T_attn": 1.337610499286294, + "Z": 2.197699714731714, + "U": 4.74111042383415, + "S": 4.331873855871101, + "F": -0.7874112291469136, + "Cv": 2.7113365849004696, + "chi": 3.961993190715313, + "xi": 3.437786905547551, + "D90": 1383, + "delta_H_90": -0.03897260917359724, + "efficiency": 0.9817767073870174, + "phase": "A (deconf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "google--gemma-2-9b-it_mongo", + "model": "google/gemma-2-9b-it", + "corpus": "mongo", + "gamma": 0.6276459084140061, + "log_A": -2.864020841938658, + "R2": 0.977314, + "theta": 10000, + "T_attn": 1.5932550289810583, + "Z": 2.5002294130110765, + "U": 5.2688459931253435, + "S": 4.22335212251707, + "F": -0.9163824928684169, + "Cv": 1.5499404060181028, + "chi": 2.6856157152473603, + "xi": 2.1469404166413715, + "D90": 1524, + "delta_H_90": -0.02993155667320758, + "efficiency": 0.6654920458725847, + "phase": "A (deconf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "google--gemma-2-9b-it_random", + "model": "google/gemma-2-9b-it", + "corpus": "random", + "gamma": 1.1347958464287666, + "log_A": -0.9640958037685541, + "R2": 0.976472, + "theta": 10000, + "T_attn": 0.8812157738742411, + "Z": 2.1948104569334257, + "U": 2.6130926338676845, + "S": 3.7514223578508843, + "F": -0.7860956906042302, + "Cv": 6.713620161112188, + "chi": 7.418626215077434, + "xi": 7.908091292052835, + "D90": 469, + "delta_H_90": -0.09964902083307159, + "efficiency": NaN, + "phase": "B (conf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "meta-llama--Llama-2-7b-hf_mongo", + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "mongo", + "gamma": 0.2870574377368437, + "log_A": -5.073101472705059, + "R2": 0.814928, + "theta": 10000, + "T_attn": 3.483623374764243, + "Z": 1.9797823294399142, + "U": 6.212383597591927, + "S": 2.4662978218000178, + "F": -0.682986904036884, + "Cv": 0.15047847898333028, + "chi": 1.4026375376238054, + "xi": 0.8012352151296246, + "D90": 1726, + "delta_H_90": -0.01711992317160403, + "efficiency": 0.34757173903600547, + "phase": "A (deconf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "meta-llama--Llama-2-7b-hf_random", + "model": "meta-llama/Llama-2-7b-hf", + "corpus": "random", + "gamma": 0.8266242679750889, + "log_A": -2.9325874169558817, + "R2": 0.993628, + "theta": 10000, + "T_attn": 1.209739465367518, + "Z": 0.893493675465163, + "U": 4.337054030641602, + "S": 3.472498090457235, + "F": 0.11261602279028816, + "Cv": 3.657194630191616, + "chi": 5.767819915282708, + "xi": 5.251962409575921, + "D90": 1254, + "delta_H_90": -0.04731642837652837, + "efficiency": 1.4292581972371223, + "phase": "A (deconf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "meta-llama--Meta-Llama-3-8B_mongo", + "model": "meta-llama/Meta-Llama-3-8B", + "corpus": "mongo", + "gamma": 1.0454762537473639, + "log_A": -2.4338207488763257, + "R2": 0.997461, + "theta": 500000, + "T_attn": 0.9565018778911902, + "Z": 0.6513010443603229, + "U": 3.0996143717814713, + "S": 2.8117899115417013, + "F": 0.4287833099298801, + "Cv": 6.160743803155553, + "chi": 21.989498201750344, + "xi": 22.485792280007733, + "D90": 718, + "delta_H_90": -0.0018095444887590362, + "efficiency": NaN, + "phase": "* Hagedorn", + "theta_eff_Pade": 501414.2135623731 + }, + { + "stem": "mistralai--Mistral-7B-v0.1_mongo", + "model": "mistralai/Mistral-7B-v0.1", + "corpus": "mongo", + "gamma": 1.060750419523944, + "log_A": -2.143867119472637, + "R2": 0.99869, + "theta": 10000, + "T_attn": 0.9427288281901332, + "Z": 0.8306737339223166, + "U": 3.0139144007271286, + "S": 3.0114927851336257, + "F": 0.18551817984693242, + "Cv": 6.28266228456109, + "chi": 16.46079167578194, + "xi": 16.95587724395123, + "D90": 674, + "delta_H_90": -0.08571614105160623, + "efficiency": NaN, + "phase": "B (conf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "mistralai--Mistral-7B-v0.1_random", + "model": "mistralai/Mistral-7B-v0.1", + "corpus": "random", + "gamma": 0.8296009929924347, + "log_A": -2.376214984270495, + "R2": 0.996923, + "theta": 10000, + "T_attn": 1.2053987500580525, + "Z": 1.5385954099870987, + "U": 4.321098335761793, + "S": 4.015657398964294, + "F": -0.43086992879835306, + "Cv": 3.6945054761026754, + "chi": 5.868578799614734, + "xi": 5.353020311304068, + "D90": 1248, + "delta_H_90": -0.04770621251295322, + "efficiency": 1.4542261163974342, + "phase": "A (deconf)", + "theta_eff_Pade": 11414.213562373096 + }, + { + "stem": "Qwen--Qwen2.5-7B_mongo", + "model": "Qwen/Qwen2.5-7B", + "corpus": "mongo", + "gamma": 0.9966953735480816, + "log_A": -2.1584093095473813, + "R2": 0.993942, + "theta": 1000000, + "T_attn": 1.0033155832159173, + "Z": 1.0045985771338626, + "U": 3.3775560256657755, + "S": 3.370982500662405, + "F": -0.004588035981881088, + "Cv": 5.704969106613326, + "chi": 302.6060629090084, + "xi": 302.105787067493, + "D90": 856, + "delta_H_90": -0.0008081142096023077, + "efficiency": 0.8546904815738186, + "phase": "* Hagedorn", + "theta_eff_Pade": 1001414.2135623731 + } +] \ No newline at end of file diff --git a/data/theta_steering/pythia-70m_theta_steering.json b/data/theta_steering/pythia-70m_theta_steering.json new file mode 100644 index 0000000000000000000000000000000000000000..eb8227da77fc6730a30625a74eae3a9a46fd2322 --- /dev/null +++ b/data/theta_steering/pythia-70m_theta_steering.json @@ -0,0 +1,177 @@ +{ + "model": "pythia-70m", + "theta_train": 10000, + "T_eval": 2000, + "thetas": [ + 2000, + 4000, + 6000, + 8000, + 10000, + 15000, + 25000, + 50000, + 100000 + ], + "distances": [ + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "sweep": { + "2000": { + "gamma_pade": 0.17157287525380988, + "gamma_obs": 0.714375409403098, + "r2": 0.9741913409622216, + "err_pct": 316.36850134166815, + "means": [ + 0.009604626725841729, + 0.004868006015506883, + 0.0028402812396557753, + 0.002274303606908587, + 0.0013922684618996246, + 0.0007004731420213121, + 0.00043012310639040964, + 0.00014885135780168135 + ] + }, + "4000": { + "gamma_pade": 0.47759225007251715, + "gamma_obs": 0.47898028877671367, + "r2": 0.6106205814695712, + "err_pct": 0.29063258542946546, + "means": [ + 0.00988029801260887, + 0.005357984657813277, + 0.003281183644301362, + 0.0022858653773760638, + 0.0010564159820256058, + 0.0006220263622127377, + 0.0002935719562098408, + 0.0023061624822730666 + ] + }, + "6000": { + "gamma_pade": 0.6185128603389075, + "gamma_obs": 0.5396399813503828, + "r2": 0.8300072248154521, + "err_pct": -12.752019245858063, + "means": [ + 0.0102943106683799, + 0.00610424934550085, + 0.0032437017890010298, + 0.0028652805979719333, + 0.0010271097721417594, + 0.0007856106406203859, + 0.0003861682948670225, + 0.0011346545764644763 + ] + }, + "8000": { + "gamma_pade": 0.6995577903553303, + "gamma_obs": 0.7449340861022274, + "r2": 0.9877581983694703, + "err_pct": 6.486425620941043, + "means": [ + 0.010118313336796643, + 0.006517327819407606, + 0.0031606309775573512, + 0.0024880319043101428, + 0.0011664263213990327, + 0.0007860678273926979, + 0.0003438393898906927, + 0.0001774132015294564 + ] + }, + "10000": { + "gamma_pade": 0.7522013138014093, + "gamma_obs": 0.7493589625280966, + "r2": 0.9868773001810779, + "err_pct": -0.3778710859926905, + "means": [ + 0.009506619756575675, + 0.0067441158721016515, + 0.0033795992821170425, + 0.002487490379927395, + 0.0014098713588383464, + 0.0007273848762755126, + 0.0002981155557234969, + 0.00018959359564745014 + ] + }, + "15000": { + "gamma_pade": 0.8276842741202114, + "gamma_obs": 0.7531976437540864, + "r2": 0.9617040039997112, + "err_pct": -8.99940142577926, + "means": [ + 0.009781096916412935, + 0.006860281446845167, + 0.0030377792962503415, + 0.002372458417853017, + 0.001678404606031513, + 0.0009876430442798461, + 0.00027079408162649087, + 0.00016501994166281028 + ] + }, + "25000": { + "gamma_pade": 0.8929202598416456, + "gamma_obs": 0.772035887056828, + "r2": 0.970148833712957, + "err_pct": -13.538092730279827, + "means": [ + 0.009533998309375926, + 0.008266664853888668, + 0.003347901162406844, + 0.002846583230469454, + 0.001913362272413603, + 0.0007265601174569584, + 0.00031707800517576996, + 0.00016776436796514342 + ] + }, + "50000": { + "gamma_pade": 0.9449874474630467, + "gamma_obs": 0.705374612315457, + "r2": 0.9161440630935891, + "err_pct": -25.356192380212512, + "means": [ + 0.009705971443973895, + 0.007247659005224705, + 0.004061414545867593, + 0.0029714647163119573, + 0.001514613861460627, + 0.000815417092644566, + 0.0009369428114748896, + 0.00013820052917736802 + ] + }, + "100000": { + "gamma_pade": 0.9721101507826946, + "gamma_obs": 0.7007617520206347, + "r2": 0.9805917850739304, + "err_pct": -27.91333868323293, + "means": [ + 0.009592079060773055, + 0.006038397048703498, + 0.004050688877598279, + 0.0029629927710630006, + 0.0013748960499166665, + 0.0008017285775470859, + 0.0004883475130959091, + 0.00020424747601484545 + ] + } + }, + "tracking_fit": { + "slope": 0.15285955209288868, + "intercept": 0.5763232410262167, + "r2": 0.1443035834215492 + } +} \ No newline at end of file diff --git a/data/training_gamma/pythia-70m_gamma_trajectory.json b/data/training_gamma/pythia-70m_gamma_trajectory.json new file mode 100644 index 0000000000000000000000000000000000000000..776d64e15338ca327de7fc9d6386e582e1d75fca --- /dev/null +++ b/data/training_gamma/pythia-70m_gamma_trajectory.json @@ -0,0 +1,162 @@ +{ + "model": "pythia-70m", + "model_id": "EleutherAI/pythia-70m", + "steps": [ + 512, + 1000, + 2000, + 4000, + 8000, + 16000, + 32000, + 64000, + 143000 + ], + "distances": [ + 10, + 20, + 50, + 100, + 200, + 500, + 1000, + 2000 + ], + "checkpoints": { + "512": { + "means": [ + 0.014567853381029434, + 0.013013510338755117, + 0.005827679155400901, + 0.0021359426128078163, + 0.0018933850177240352, + 0.00037104320920459743, + 0.00017867149841587583, + 0.0004218808321537735 + ], + "gamma": 0.8550246564058653, + "r2": 0.9160516162848862, + "n_pts": 8 + }, + "1000": { + "means": [ + 0.01347908391099837, + 0.009875446363326372, + 0.003748049277540607, + 0.0015054708201205358, + 0.0013264099975559575, + 0.00044005458839314486, + 0.00019404526192122118, + 0.0002246908314014541 + ], + "gamma": 0.8565449645184069, + "r2": 0.9751622510551018, + "n_pts": 8 + }, + "2000": { + "means": [ + 0.012450078150464427, + 0.009568355440084513, + 0.003957201376033481, + 0.0019484649956413905, + 0.0011999284352997267, + 0.00044975969183522267, + 0.00017231862647501357, + 0.00014923718247498782 + ], + "gamma": 0.9047541091805307, + "r2": 0.9885252839347416, + "n_pts": 8 + }, + "4000": { + "means": [ + 0.012122047643384173, + 0.008707450895518479, + 0.0036634952345694833, + 0.0014206855683773435, + 0.0011455025599894325, + 0.0003734741826873586, + 0.0001456256307326006, + 7.454777513089539e-05 + ], + "gamma": 0.9831795444940423, + "r2": 0.9886665650550166, + "n_pts": 8 + }, + "8000": { + "means": [ + 0.013803851643266777, + 0.010699979349091233, + 0.004052742326974921, + 0.001419275840150173, + 0.001004462761152859, + 0.00033230642811506384, + 0.00013103776577610106, + 5.6560009842441004e-05 + ], + "gamma": 1.0635727794689445, + "r2": 0.9896715614987192, + "n_pts": 8 + }, + "16000": { + "means": [ + 0.012672862317413092, + 0.010971801272696919, + 0.0041887151449272, + 0.0017119876536728245, + 0.0009156849272989753, + 0.0003656817950715979, + 0.0001601850737194531, + 8.173211619527539e-05 + ], + "gamma": 1.0019987105051475, + "r2": 0.9917886961639084, + "n_pts": 8 + }, + "32000": { + "means": [ + 0.010684995549834438, + 0.009041740679984084, + 0.0035101745460956917, + 0.001763116369258771, + 0.0011439746846008344, + 0.0004293754894206359, + 0.00025329675633037774, + 0.00010552197128390615 + ], + "gamma": 0.8865139876578815, + "r2": 0.9912202870364645, + "n_pts": 8 + }, + "64000": { + "means": [ + 0.009941841238954413, + 0.00816984858154481, + 0.004024662110249564, + 0.0019963142813746045, + 0.0014300855282676317, + 0.0005831896253312152, + 0.0003030837157666639, + 0.00017063526470590458 + ], + "gamma": 0.7947712889883813, + "r2": 0.9907986737968195, + "n_pts": 8 + }, + "143000": { + "means": [ + 0.009506619756575675, + 0.008156256267749188, + 0.0036576768236247716, + 0.0018867265357079709, + 0.0014858170062628537, + 0.0006240812543056665, + 0.00031680072714961065, + 0.0001656838764601629 + ], + "gamma": 0.7805265046347023, + "r2": 0.9881754004185676, + "n_pts": 8 + } + } +} \ No newline at end of file diff --git a/data/val_a_mi5_pythia/pythia-1b.json b/data/val_a_mi5_pythia/pythia-1b.json new file mode 100644 index 0000000000000000000000000000000000000000..9598f480bb54dd11d3e47096e455aaeded2847c7 --- /dev/null +++ b/data/val_a_mi5_pythia/pythia-1b.json @@ -0,0 +1,36 @@ +{ + "model": "pythia-1b", + "N": 16, + "L_crit": { + "cos_mean": -0.0004907500697299838, + "cos_std": 0.07011894881725311, + "frob_ratio": 0.3525602820076393, + "theory_ratio": 0.35355339059327373, + "frob_ratio_dev": 0.0009931085856344501, + "SR_sum": 58.71854019165039 + }, + "n_heads": 8, + "D": 2048, + "PE": "RoPE", + "L0": { + "cos_mean": -0.03010750189423561, + "cos_std": 0.16891370713710785, + "frob_ratio": 0.3420823254108457, + "theory_ratio": 0.35355339059327373, + "frob_ratio_dev": 0.01147106518242802, + "SR_sum": 78.78228759765625 + }, + "L_mid": { + "cos_mean": -0.0013874734286218882, + "cos_std": 0.0860472321510315, + "frob_ratio": 0.3046497781235215, + "theory_ratio": 0.35355339059327373, + "frob_ratio_dev": 0.04890361246975222, + "SR_sum": 87.7594985961914 + }, + "R_joint": { + "mean": 0.11420554241727623, + "ratio": 1.0447954329630256, + "interp": "ADITIVO" + } +} \ No newline at end of file diff --git a/data/val_a_mi5_pythia/pythia-2.8b.json b/data/val_a_mi5_pythia/pythia-2.8b.json new file mode 100644 index 0000000000000000000000000000000000000000..1a42193beb29d16dbdb39de3037202aae0e77e4f --- /dev/null +++ b/data/val_a_mi5_pythia/pythia-2.8b.json @@ -0,0 +1,36 @@ +{ + "model": "pythia-2.8b", + "N": 32, + "L_crit": { + "cos_mean": 0.003106344025582075, + "cos_std": 0.03716026246547699, + "frob_ratio": 0.17803061426258854, + "theory_ratio": 0.17677669529663687, + "frob_ratio_dev": 0.0012539189659516725, + "SR_sum": 235.3758087158203 + }, + "n_heads": 32, + "D": 2560, + "PE": "RoPE", + "L0": { + "cos_mean": -0.010402541607618332, + "cos_std": 0.06438879668712616, + "frob_ratio": 0.17232778398984647, + "theory_ratio": 0.17677669529663687, + "frob_ratio_dev": 0.0044489113067903985, + "SR_sum": 43.436824798583984 + }, + "L_mid": { + "cos_mean": 0.0022677190136164427, + "cos_std": 0.036118753254413605, + "frob_ratio": 0.1725754614124892, + "theory_ratio": 0.17677669529663687, + "frob_ratio_dev": 0.00420123388414767, + "SR_sum": 184.76266479492188 + }, + "R_joint": { + "mean": 0.02465852069764694, + "ratio": 0.9823947791222547, + "interp": "ADITIVO" + } +} \ No newline at end of file diff --git a/data/val_a_mi5_pythia/pythia-70m.json b/data/val_a_mi5_pythia/pythia-70m.json new file mode 100644 index 0000000000000000000000000000000000000000..b5fa9ea080cdaf2be9787b0d35345bae6e93ec69 --- /dev/null +++ b/data/val_a_mi5_pythia/pythia-70m.json @@ -0,0 +1,36 @@ +{ + "model": "pythia-70m", + "N": 6, + "L_crit": { + "cos_mean": -0.009147935546934605, + "cos_std": 0.08791883289813995, + "frob_ratio": 0.45632481122012086, + "theory_ratio": 0.35355339059327373, + "frob_ratio_dev": 0.10277142062684713, + "SR_sum": 38.33124542236328 + }, + "n_heads": 8, + "D": 512, + "PE": "RoPE", + "L0": { + "cos_mean": -0.05857821926474571, + "cos_std": 0.16157503426074982, + "frob_ratio": 0.33038856582686504, + "theory_ratio": 0.35355339059327373, + "frob_ratio_dev": 0.023164824766408687, + "SR_sum": 15.625130653381348 + }, + "L_mid": { + "cos_mean": -0.00010677547106752172, + "cos_std": 0.06058668717741966, + "frob_ratio": 0.4611839994469595, + "theory_ratio": 0.35355339059327373, + "frob_ratio_dev": 0.10763060885368575, + "SR_sum": 22.862993240356445 + }, + "R_joint": { + "mean": -0.006075141929855642, + "ratio": 0.20052976451614093, + "interp": "SUBLINEAL" + } +} \ No newline at end of file diff --git a/data/val_b_h3_gptj/gpt-j-6b.json b/data/val_b_h3_gptj/gpt-j-6b.json new file mode 100644 index 0000000000000000000000000000000000000000..a29c1029f3d6e07843fac56f06790224b684058a --- /dev/null +++ b/data/val_b_h3_gptj/gpt-j-6b.json @@ -0,0 +1,119 @@ +{ + "model": "EleutherAI/gpt-j-6b", + "N": 28, + "L_crit_pred": 27, + "PE": "AbsPE", + "alpha_pred": 0.9464285714285714, + "per_seed": { + "seed42": { + "L_crit_90": 24, + "L_crit_99": 26, + "max_R": 0.9999372451848154, + "mean_R": { + "0": 0.01786466915355279, + "1": -0.010527140520682425, + "2": 0.03854757944914213, + "3": 0.08274670595482667, + "4": 0.13675020074325966, + "5": 0.1484338797849876, + "6": 0.17472909949393195, + "7": 0.2135127147580879, + "8": 0.251070054368994, + "9": 0.25261406189825536, + "10": 0.2810775351303979, + "11": 0.2993857015518105, + "12": 0.32588944877573467, + "13": 0.3531775909657833, + "14": 0.3722919756030284, + "15": 0.47497462621026687, + "16": 0.5589184229588625, + "17": 0.6021839883073105, + "18": 0.7034701572356389, + "19": 0.7816097512187029, + "20": 0.8076077467829, + "21": 0.8229127471877202, + "22": 0.8466824636584254, + "23": 0.8953034382857665, + "24": 0.9591147356418325, + "25": 0.9818113347694511, + "26": 0.9930652664109139, + "27": 0.9999372451848154 + }, + "n_valid": 30 + }, + "seed7": { + "L_crit_90": 25, + "L_crit_99": 27, + "max_R": 1.0001540382703735, + "mean_R": { + "0": -0.0061837089841554425, + "1": -0.0005551504409134023, + "2": -0.007211238988701085, + "3": 0.03985424513268078, + "4": 0.1016883611219925, + "5": 0.13181716072117985, + "6": 0.15168512372687828, + "7": 0.16295116085814848, + "8": 0.16155987198322952, + "9": 0.16303091131509545, + "10": 0.18272343890533368, + "11": 0.17365021305546086, + "12": 0.17285196010566167, + "13": 0.1863368221556549, + "14": 0.181451933891328, + "15": 0.27610376041016554, + "16": 0.3456844652691243, + "17": 0.40633114193748016, + "18": 0.47310261866378756, + "19": 0.5621650503255469, + "20": 0.6168175522321068, + "21": 0.637324216388248, + "22": 0.6542284707492888, + "23": 0.7244391535515716, + "24": 0.8732869636108217, + "25": 0.9078484070225398, + "26": 0.9850902085224299, + "27": 1.0001540382703735 + }, + "n_valid": 30 + }, + "seed123": { + "L_crit_90": 24, + "L_crit_99": 27, + "max_R": 0.9997789589807674, + "mean_R": { + "0": 0.005270975023480817, + "1": -0.0019103258877417322, + "2": -0.02213425410706478, + "3": -0.00642247846890937, + "4": 0.15963506561880564, + "5": 0.1662498583863705, + "6": 0.1829459136458766, + "7": 0.19400895617503283, + "8": 0.2250015960012758, + "9": 0.2594336520621531, + "10": 0.2948331436491046, + "11": 0.3211851785062886, + "12": 0.3664570129947402, + "13": 0.39666032851436533, + "14": 0.38009036547225605, + "15": 0.45236774978459443, + "16": 0.5268497309316847, + "17": 0.5338022103701924, + "18": 0.6190035795783098, + "19": 0.6855548205716219, + "20": 0.7507048138393922, + "21": 0.7622172907154162, + "22": 0.7441759335484132, + "23": 0.8199289314045264, + "24": 0.9419827423548127, + "25": 0.9489644410796654, + "26": 0.9729383880878635, + "27": 0.9997789589807674 + }, + "n_valid": 30 + } + }, + "L_crit_99_mean": 26.666666666666668, + "L_crit_90_mean": 24.333333333333332 +} \ No newline at end of file diff --git a/data/val_c_h3_rope/Mistral-7B.json b/data/val_c_h3_rope/Mistral-7B.json new file mode 100644 index 0000000000000000000000000000000000000000..2e69692129ba1c602461670a897e7f52f4b87f8b --- /dev/null +++ b/data/val_c_h3_rope/Mistral-7B.json @@ -0,0 +1,96 @@ +{ + "model": "Mistral-7B", + "N": 32, + "theta": 10000, + "T_eff": 4096, + "d_head": 128, + "alpha_pred": 0.8595864520335161, + "L_crit_pred": 28, + "PE": "RoPE", + "per_seed": { + "seed42": { + "L90": 30, + "L99": 31, + "maxR": 0.9999999951495564, + "mean_R": { + "0": 0.006680211491536203, + "1": -0.004598894406640415, + "2": -0.027109134693773167, + "3": -0.07271501272848102, + "4": -0.12526608511871465, + "5": -0.12712785805454893, + "6": -0.13667031100913543, + "7": -0.10541089021141084, + "8": -0.11418099386182216, + "9": -0.07758169474144672, + "10": -0.07431989920025127, + "11": -0.0536738841048624, + "12": -0.07420474905329233, + "13": -0.028331352691248944, + "14": -0.056251631328113325, + "15": 0.006485263336282021, + "16": 0.006594133011862962, + "17": 0.06110445909732509, + "18": 0.24353191880728944, + "19": 0.27959150271858124, + "20": 0.3402229716512159, + "21": 0.3560350970416213, + "22": 0.4059399653449839, + "23": 0.42652971208887414, + "24": 0.4889374632009395, + "25": 0.4759502121525439, + "26": 0.5730592316723044, + "27": 0.5757327271402719, + "28": 0.6760352934572866, + "29": 0.8111360614216274, + "30": 0.9632001876443124, + "31": 0.9999999951495564 + }, + "n_valid": 20 + }, + "seed7": { + "L90": 29, + "L99": 30, + "maxR": 1.0312112731653884, + "mean_R": { + "0": 0.015844672802510316, + "1": -0.031690724405947314, + "2": -0.02196939126017604, + "3": -0.04896898178235286, + "4": -0.03633698671456514, + "5": -0.03591292931352553, + "6": -0.027151804207535317, + "7": -0.059121750115265895, + "8": -0.043643949896272426, + "9": 0.00040150891778559933, + "10": 0.034376353507370276, + "11": 0.05767965031719964, + "12": 0.2370576122500178, + "13": 0.2541769597620215, + "14": 0.2607525089571364, + "15": 0.3008829916683736, + "16": 0.30340807777005874, + "17": 0.3714095269806633, + "18": 0.5717811105343873, + "19": 0.5815312035985347, + "20": 0.5974020288937357, + "21": 0.6284787454496517, + "22": 0.6726815346162757, + "23": 0.6711120014312456, + "24": 0.7342683895853845, + "25": 0.7252188449228382, + "26": 0.7716334274337351, + "27": 0.7885770650139896, + "28": 0.8419872827039843, + "29": 0.9199706569356085, + "30": 1.0312112731653884, + "31": 0.9999999922117027 + }, + "n_valid": 20 + } + }, + "L99_mean": 30.5, + "L90_mean": 29.5, + "error_vs_pred": 2.5, + "verdict": "DEVIATED" +} \ No newline at end of file diff --git a/data/val_c_h3_rope/Qwen2.5-7B.json b/data/val_c_h3_rope/Qwen2.5-7B.json new file mode 100644 index 0000000000000000000000000000000000000000..24cd7d5878c9b2311eeb44e64b272779aa45976b --- /dev/null +++ b/data/val_c_h3_rope/Qwen2.5-7B.json @@ -0,0 +1,88 @@ +{ + "model": "Qwen2.5-7B", + "N": 28, + "theta": 1000000, + "T_eff": 32768, + "d_head": 128, + "alpha_pred": 0.7569558207385388, + "L_crit_pred": 21, + "PE": "RoPE", + "per_seed": { + "seed42": { + "L90": 26, + "L99": 27, + "maxR": 1.000247954206381, + "mean_R": { + "0": 0.013038125401253653, + "1": 0.01114896313089698, + "2": 0.02108781907481502, + "3": 0.03899546152161319, + "4": 0.04266129880660459, + "5": 0.062062592249048754, + "6": 0.05966961759708802, + "7": 0.0728271873794919, + "8": 0.08459936531735068, + "9": 0.08600735578547583, + "10": 0.09200806608519382, + "11": 0.0966937233393409, + "12": 0.1049464050282117, + "13": 0.11926519584765445, + "14": 0.15820101618781174, + "15": 0.1799043574464228, + "16": 0.19914624124142527, + "17": 0.2439881935740295, + "18": 0.2716958300739539, + "19": 0.3031028684752916, + "20": 0.3255157265287226, + "21": 0.34914071982089023, + "22": 0.6790380378746905, + "23": 0.7564122807760588, + "24": 0.8231928989373116, + "25": 0.8523312428720133, + "26": 0.9531653232182503, + "27": 1.000247954206381 + }, + "n_valid": 20 + }, + "seed7": { + "L90": 26, + "L99": 27, + "maxR": 1.0000716525798699, + "mean_R": { + "0": -0.01102028764421317, + "1": 0.010273362452771363, + "2": 0.026230224049377165, + "3": 0.03473231762862085, + "4": 0.03704287594876869, + "5": 0.05765849436379038, + "6": 0.05046219464167971, + "7": 0.05573622871980131, + "8": 0.060045490313325164, + "9": 0.06017900941925912, + "10": 0.060852109360184446, + "11": 0.06759077721926929, + "12": 0.07122506138307064, + "13": 0.0766346145317739, + "14": 0.11453427438255608, + "15": 0.12245065878310679, + "16": 0.13236582003922298, + "17": 0.1793415916185534, + "18": 0.20136772305716022, + "19": 0.21603635123807088, + "20": 0.25291851403389176, + "21": 0.27849403277484075, + "22": 0.5872968504881506, + "23": 0.6609584009666459, + "24": 0.7346525485243174, + "25": 0.7793978694102143, + "26": 0.931798757984984, + "27": 1.0000716525798699 + }, + "n_valid": 20 + } + }, + "L99_mean": 27.0, + "L90_mean": 26.0, + "error_vs_pred": 6.0, + "verdict": "DEVIATED" +} \ No newline at end of file diff --git a/data/val_d_etask_rope/pythia-160m.json b/data/val_d_etask_rope/pythia-160m.json new file mode 100644 index 0000000000000000000000000000000000000000..bfdb136863fba6875867351c5927264e12c3cdba --- /dev/null +++ b/data/val_d_etask_rope/pythia-160m.json @@ -0,0 +1,57 @@ +{ + "model": "pythia-160m", + "N": 12, + "L_crit": 3, + "N_sem": 1, + "tipo": "I", + "PE": "RoPE", + "max_gap": 0.7586698849995931, + "d_half_gap": 500, + "by_distance": { + "1": { + "gap": 0.0, + "acc": 0.03333333333333333, + "n": 30 + }, + "5": { + "gap": 0.022303160031636557, + "acc": 0.03333333333333333, + "n": 30 + }, + "10": { + "gap": -0.02145047187805176, + "acc": 0.03333333333333333, + "n": 30 + }, + "20": { + "gap": 0.15865719318389893, + "acc": 0.03333333333333333, + "n": 30 + }, + "50": { + "gap": 0.041527525583903, + "acc": 0.03333333333333333, + "n": 30 + }, + "100": { + "gap": 0.02176191012064616, + "acc": 0.1, + "n": 30 + }, + "200": { + "gap": 0.17015668551127117, + "acc": 0.0, + "n": 30 + }, + "500": { + "gap": 0.43396586577097573, + "acc": 0.06666666666666667, + "n": 30 + }, + "1000": { + "gap": 0.7586698849995931, + "acc": 0.03333333333333333, + "n": 30 + } + } +} \ No newline at end of file diff --git a/data/val_d_etask_rope/pythia-1b.json b/data/val_d_etask_rope/pythia-1b.json new file mode 100644 index 0000000000000000000000000000000000000000..65d566210f1fd6e016f20d20dd6d4cdca788fb0c --- /dev/null +++ b/data/val_d_etask_rope/pythia-1b.json @@ -0,0 +1,57 @@ +{ + "model": "pythia-1b", + "N": 16, + "L_crit": 15, + "N_sem": 1, + "tipo": "I", + "PE": "RoPE", + "max_gap": 1.7335802714029949, + "d_half_gap": 100, + "by_distance": { + "1": { + "gap": 0.0, + "acc": 0.03333333333333333, + "n": 30 + }, + "5": { + "gap": -0.003004177411397298, + "acc": 0.0, + "n": 30 + }, + "10": { + "gap": 0.11643293698628744, + "acc": 0.0, + "n": 30 + }, + "20": { + "gap": 0.47503121693929035, + "acc": 0.0, + "n": 30 + }, + "50": { + "gap": 0.4946670214335124, + "acc": 0.0, + "n": 30 + }, + "100": { + "gap": 0.9640305836995443, + "acc": 0.0, + "n": 30 + }, + "200": { + "gap": 1.25825834274292, + "acc": 0.0, + "n": 30 + }, + "500": { + "gap": 1.6271676381429037, + "acc": 0.0, + "n": 30 + }, + "1000": { + "gap": 1.7335802714029949, + "acc": 0.0, + "n": 30 + } + } +} \ No newline at end of file diff --git a/data/val_d_etask_rope/pythia-70m.json b/data/val_d_etask_rope/pythia-70m.json new file mode 100644 index 0000000000000000000000000000000000000000..303c633f36c53d87c692bdeca582f4894b389f8d --- /dev/null +++ b/data/val_d_etask_rope/pythia-70m.json @@ -0,0 +1,57 @@ +{ + "model": "pythia-70m", + "N": 6, + "L_crit": 4, + "N_sem": 1, + "tipo": "I", + "PE": "RoPE", + "max_gap": 0.4202185312906901, + "d_half_gap": 5, + "by_distance": { + "1": { + "gap": 0.0, + "acc": 0.03333333333333333, + "n": 30 + }, + "5": { + "gap": 0.32323214213053386, + "acc": 0.03333333333333333, + "n": 30 + }, + "10": { + "gap": 0.4202185312906901, + "acc": 0.0, + "n": 30 + }, + "20": { + "gap": 0.23320541381835938, + "acc": 0.0, + "n": 30 + }, + "50": { + "gap": 0.120762570699056, + "acc": 0.0, + "n": 30 + }, + "100": { + "gap": 0.21337209542592367, + "acc": 0.0, + "n": 30 + }, + "200": { + "gap": 0.2684545755386353, + "acc": 0.03333333333333333, + "n": 30 + }, + "500": { + "gap": 0.16361497243245443, + "acc": 0.0, + "n": 30 + }, + "1000": { + "gap": 0.36503542264302574, + "acc": 0.03333333333333333, + "n": 30 + } + } +} \ No newline at end of file diff --git a/data/wo_latency/wo_latency.json b/data/wo_latency/wo_latency.json new file mode 100644 index 0000000000000000000000000000000000000000..2bc12fcc52cc3864e137e6a3ce4ab2d12a71a48b --- /dev/null +++ b/data/wo_latency/wo_latency.json @@ -0,0 +1,53 @@ +{ + "model": "EleutherAI/pythia-70m", + "k_dead": 23, + "dead_dims_per_head": 46, + "d_head": 64, + "n_zeroed_total": 2208, + "ppl_baseline": 152764464.35736594, + "ppl_pruned": 1063185276.825252, + "baseline": { + "128": { + "tokens_per_sec": 32570.637569949966, + "ms_per_token": 0.030702500000264383 + }, + "256": { + "tokens_per_sec": 65351.96885426902, + "ms_per_token": 0.015301757812835604 + }, + "512": { + "tokens_per_sec": 71022.82579284054, + "ms_per_token": 0.014079980468769312 + }, + "1024": { + "tokens_per_sec": 76560.36605362123, + "ms_per_token": 0.013061588541773972 + }, + "2048": { + "tokens_per_sec": 72810.11491772621, + "ms_per_token": 0.01373435546874191 + } + }, + "pruned": { + "128": { + "tokens_per_sec": 32135.93500479283, + "ms_per_token": 0.031117812500269793 + }, + "256": { + "tokens_per_sec": 63828.93844526313, + "ms_per_token": 0.015666874999927433 + }, + "512": { + "tokens_per_sec": 70990.39457336621, + "ms_per_token": 0.01408641276062402 + }, + "1024": { + "tokens_per_sec": 76246.35614665123, + "ms_per_token": 0.013115380859337241 + }, + "2048": { + "tokens_per_sec": 72812.34112301709, + "ms_per_token": 0.01373393554686686 + } + } +} \ No newline at end of file diff --git a/data/zipf/zipf_analysis.json b/data/zipf/zipf_analysis.json new file mode 100644 index 0000000000000000000000000000000000000000..76a803dcf47bba2392a9fd0374fb05b0be01462c --- /dev/null +++ b/data/zipf/zipf_analysis.json @@ -0,0 +1,90 @@ +[ + { + "model": "EleutherAI/pythia-70m", + "total_tokens": 169329, + "vocab_used": 17324, + "alpha_zipf": 0.9973645096319834, + "zipf_R2": 0.978809, + "gamma_mongo": 0.7476017873166874, + "gamma_random": null, + "delta_gamma": null, + "alpha_vs_gamma_mongo": 0.24976272231529606, + "alpha_vs_gamma_random": null + }, + { + "model": "EleutherAI/pythia-1b", + "total_tokens": 169329, + "vocab_used": 17324, + "alpha_zipf": 0.9973645096319834, + "zipf_R2": 0.978809, + "gamma_mongo": 0.9311078627189842, + "gamma_random": null, + "delta_gamma": null, + "alpha_vs_gamma_mongo": 0.06625664691299926, + "alpha_vs_gamma_random": null + }, + { + "model": "EleutherAI/pythia-1.4b", + "total_tokens": 169329, + "vocab_used": 17324, + "alpha_zipf": 0.9973645096319834, + "zipf_R2": 0.978809, + "gamma_mongo": 0.7050725013322717, + "gamma_random": null, + "delta_gamma": null, + "alpha_vs_gamma_mongo": 0.2922920082997117, + "alpha_vs_gamma_random": null + }, + { + "model": "meta-llama/Llama-2-7b-hf", + "total_tokens": 200246, + "vocab_used": 11341, + "alpha_zipf": 1.2322830988294957, + "zipf_R2": 0.927161, + "gamma_mongo": 0.2870574377368437, + "gamma_random": 0.8266242679750889, + "delta_gamma": -0.5395668302382453, + "alpha_vs_gamma_mongo": 0.9452256610926519, + "alpha_vs_gamma_random": 0.4056588308544068 + }, + { + "model": "mistralai/Mistral-7B-v0.1", + "total_tokens": 193925, + "vocab_used": 12353, + "alpha_zipf": 1.1648981055146908, + "zipf_R2": 0.939854, + "gamma_mongo": 1.060750419523944, + "gamma_random": 0.8296009929924347, + "delta_gamma": 0.23114942653150938, + "alpha_vs_gamma_mongo": 0.10414768599074664, + "alpha_vs_gamma_random": 0.335297112522256 + }, + { + "model": "google/gemma-2-9b-it", + "error": "You are trying to access a gated repo.\nMake sure to have access to it at https://huggingface.co/google/gemma-2-9b-it.\n401 Client Error. (Request ID: Root=1-69e770ce-6778cc954cb65267276b20ff;f52fb9d9-3b4a-484c-9d89-1a7f614879c1)\n\nCannot access gated repo for url https://huggingface.co/google/gemma-2-9b-it/resolve/main/config.json.\nAccess to model google/gemma-2-9b-it is restricted. You must have access to it and be authenticated to access it. Please log in." + }, + { + "model": "Qwen/Qwen2.5-7B", + "total_tokens": 172878, + "vocab_used": 17271, + "alpha_zipf": 1.0155457637932153, + "zipf_R2": 0.979609, + "gamma_mongo": 0.9966953735480816, + "gamma_random": null, + "delta_gamma": null, + "alpha_vs_gamma_mongo": 0.018850390245133686, + "alpha_vs_gamma_random": null + }, + { + "model": "meta-llama/Meta-Llama-3-8B", + "total_tokens": 170402, + "vocab_used": 17622, + "alpha_zipf": 1.0072810259559108, + "zipf_R2": 0.977575, + "gamma_mongo": 1.0454762537473639, + "gamma_random": null, + "delta_gamma": null, + "alpha_vs_gamma_mongo": -0.03819522779145301, + "alpha_vs_gamma_random": null + } +] \ No newline at end of file diff --git a/data/zipf/zipf_gamma_correlation.csv b/data/zipf/zipf_gamma_correlation.csv new file mode 100644 index 0000000000000000000000000000000000000000..4153d51edaa73634490e63ac60eab10688e23c00 --- /dev/null +++ b/data/zipf/zipf_gamma_correlation.csv @@ -0,0 +1,8 @@ +model,alpha_zipf,zipf_R2,gamma_mongo,gamma_random,delta_gamma,alpha_minus_gamma_mongo +EleutherAI/pythia-70m,0.997365,0.9788,0.7476017873166874,,,0.24976272231529606 +EleutherAI/pythia-1b,0.997365,0.9788,0.9311078627189842,,,0.06625664691299926 +EleutherAI/pythia-1.4b,0.997365,0.9788,0.7050725013322717,,,0.2922920082997117 +meta-llama/Llama-2-7b-hf,1.232283,0.9272,0.2870574377368437,0.8266242679750889,-0.5395668302382453,0.9452256610926519 +mistralai/Mistral-7B-v0.1,1.164898,0.9399,1.060750419523944,0.8296009929924347,0.23114942653150938,0.10414768599074664 +Qwen/Qwen2.5-7B,1.015546,0.9796,0.9966953735480816,,,0.018850390245133686 +meta-llama/Meta-Llama-3-8B,1.007281,0.9776,1.0454762537473639,,,-0.03819522779145301