anshdadhich
/

richneuron-vs-vanilla-benchmark

Model card Files Files and versions

xet

Community

anshdadhich commited on Apr 24

Commit

2e1b775

verified ·

1 Parent(s): 16b1a41

v13 results

Browse files

Files changed (1) hide show

results_v13.json +462 -0

results_v13.json ADDED Viewed

	@@ -0,0 +1,462 @@

+{
+  "tasks": {
+    "Complex": {
+      "Vanilla": {
+        "mean": 0.0574857605000337,
+        "std": 0.008335319220438129,
+        "scores": [
+          0.0631193146109581,
+          0.06363634765148163,
+          0.04570161923766136
+        ],
+        "params": 4993,
+        "hidden": 48
+      },
+      "SinGLU": {
+        "mean": 0.014336802531033754,
+        "std": 0.011309525106379352,
+        "scores": [
+          0.03023100085556507,
+          0.007935304194688797,
+          0.004844102542847395
+        ],
+        "params": 4811,
+        "hidden": 31
+      },
+      "v10:free": {
+        "mean": 0.008007933618500829,
+        "std": 0.0033759458898304736,
+        "scores": [
+          0.003235818585380912,
+          0.010519308969378471,
+          0.010268673300743103
+        ],
+        "params": 4915,
+        "hidden": 27
+      },
+      "v12:FM": {
+        "mean": 0.00848957539225618,
+        "std": 0.003177204089455024,
+        "scores": [
+          0.00399657292291522,
+          0.010776516050100327,
+          0.010695637203752995
+        ],
+        "params": 4915,
+        "hidden": 27
+      },
+      "v13": {
+        "mean": 0.007401840606083472,
+        "std": 0.002616566819161181,
+        "scores": [
+          0.004813301842659712,
+          0.006406080909073353,
+          0.010986139066517353
+        ],
+        "params": 4915,
+        "hidden": 27
+      }
+    },
+    "Nested": {
+      "Vanilla": {
+        "mean": 0.0486922413110733,
+        "std": 0.0030859308547071134,
+        "scores": [
+          0.0464077852666378,
+          0.05305477976799011,
+          0.046614158898591995
+        ],
+        "params": 2961,
+        "hidden": 37
+      },
+      "SinGLU": {
+        "mean": 0.00021879897879747054,
+        "std": 9.755789869089583e-05,
+        "scores": [
+          8.296288433484733e-05,
+          0.00030763907125219703,
+          0.00026579498080536723
+        ],
+        "params": 2993,
+        "hidden": 24
+      },
+      "v10:free": {
+        "mean": 0.0004357353803546478,
+        "std": 0.0002821467022311389,
+        "scores": [
+          0.00018595167784951627,
+          0.00029115326469764113,
+          0.0008301011985167861
+        ],
+        "params": 2983,
+        "hidden": 21
+      },
+      "v12:FM": {
+        "mean": 0.0006872733841494968,
+        "std": 0.0007569946594791463,
+        "scores": [
+          0.00018634692241903394,
+          0.00011836741759907454,
+          0.0017571058124303818
+        ],
+        "params": 2983,
+        "hidden": 21
+      },
+      "v13": {
+        "mean": 0.0009911912444901343,
+        "std": 0.001121030724883482,
+        "scores": [
+          0.00018693839956540614,
+          0.00021012374782003462,
+          0.002576511586084962
+        ],
+        "params": 2983,
+        "hidden": 21
+      }
+    },
+    "Spiral": {
+      "Vanilla": {
+        "mean": 0.851111114025116,
+        "std": 0.12812993960807575,
+        "scores": [
+          0.9366666674613953,
+          0.6700000166893005,
+          0.9466666579246521
+        ],
+        "params": 2999,
+        "hidden": 37
+      },
+      "SinGLU": {
+        "mean": 0.4422222177187602,
+        "std": 0.020427534068557608,
+        "scores": [
+          0.46666666865348816,
+          0.4166666567325592,
+          0.44333332777023315
+        ],
+        "params": 3018,
+        "hidden": 24
+      },
+      "v10:free": {
+        "mean": 0.9922222296396891,
+        "std": 0.005665571834209818,
+        "scores": [
+          0.9866666793823242,
+          0.9900000095367432,
+          1.0
+        ],
+        "params": 3005,
+        "hidden": 21
+      },
+      "v12:FM": {
+        "mean": 0.8666666547457377,
+        "std": 0.01962614353715491,
+        "scores": [
+          0.8733333349227905,
+          0.8866666555404663,
+          0.8399999737739563
+        ],
+        "params": 3005,
+        "hidden": 21
+      },
+      "v13": {
+        "mean": 0.46888887882232666,
+        "std": 0.013425607810265063,
+        "scores": [
+          0.47999998927116394,
+          0.44999998807907104,
+          0.476666659116745
+        ],
+        "params": 3005,
+        "hidden": 21
+      }
+    },
+    "Checker": {
+      "Vanilla": {
+        "mean": 0.5788888931274414,
+        "std": 0.022825147689377753,
+        "scores": [
+          0.596666693687439,
+          0.54666668176651,
+          0.5933333039283752
+        ],
+        "params": 2999,
+        "hidden": 37
+      },
+      "SinGLU": {
+        "mean": 0.9377777775128683,
+        "std": 0.014989694108291244,
+        "scores": [
+          0.9166666865348816,
+          0.9466666579246521,
+          0.949999988079071
+        ],
+        "params": 3018,
+        "hidden": 24
+      },
+      "v10:free": {
+        "mean": 0.9377777775128683,
+        "std": 0.019116260140502127,
+        "scores": [
+          0.9133333563804626,
+          0.9599999785423279,
+          0.9399999976158142
+        ],
+        "params": 3005,
+        "hidden": 21
+      },
+      "v12:FM": {
+        "mean": 0.9444444378217062,
+        "std": 0.01257077523265727,
+        "scores": [
+          0.9266666769981384,
+          0.95333331823349,
+          0.95333331823349
+        ],
+        "params": 3005,
+        "hidden": 21
+      },
+      "v13": {
+        "mean": 0.9477777679761251,
+        "std": 0.01342559383367107,
+        "scores": [
+          0.9366666674613953,
+          0.9399999976158142,
+          0.9666666388511658
+        ],
+        "params": 3005,
+        "hidden": 21
+      }
+    },
+    "HiFreq": {
+      "Vanilla": {
+        "mean": 1.099808136622111,
+        "std": 0.01846947085143113,
+        "scores": [
+          1.0741740465164185,
+          1.108283281326294,
+          1.1169670820236206
+        ],
+        "params": 7999,
+        "hidden": 62
+      },
+      "SinGLU": {
+        "mean": 1.0171122153600056,
+        "std": 0.06186627648533991,
+        "scores": [
+          1.0643826723098755,
+          0.9297174215316772,
+          1.0572365522384644
+        ],
+        "params": 8214,
+        "hidden": 41
+      },
+      "v10:free": {
+        "mean": 1.223051091035207,
+        "std": 0.2257838672540006,
+        "scores": [
+          1.2578459978103638,
+          1.480534553527832,
+          0.9307727217674255
+        ],
+        "params": 7734,
+        "hidden": 35
+      },
+      "v12:FM": {
+        "mean": 2.0235122044881186,
+        "std": 0.666276192631495,
+        "scores": [
+          2.5847370624542236,
+          1.0874183177947998,
+          2.398381233215332
+        ],
+        "params": 7734,
+        "hidden": 35
+      },
+      "v13": {
+        "mean": 2.4426697889963784,
+        "std": 0.21200546216452884,
+        "scores": [
+          2.7208240032196045,
+          2.400505781173706,
+          2.206679582595825
+        ],
+        "params": 7734,
+        "hidden": 35
+      }
+    },
+    "Memorize": {
+      "Vanilla": {
+        "mean": 0.15677066644032797,
+        "std": 0.010408440580230162,
+        "scores": [
+          0.1653069108724594,
+          0.14211732149124146,
+          0.16288776695728302
+        ],
+        "params": 4926,
+        "hidden": 46
+      },
+      "SinGLU": {
+        "mean": 9.313488410119438e-07,
+        "std": 1.0663730164468857e-06,
+        "scores": [
+          1.7445064770527097e-07,
+          1.801714688554057e-07,
+          2.4394244064751547e-06
+        ],
+        "params": 5067,
+        "hidden": 31
+      },
+      "v10:free": {
+        "mean": 1.687507161089267e-05,
+        "std": 1.2195058888229233e-05,
+        "scores": [
+          2.187806967413053e-05,
+          7.998311701840066e-08,
+          2.8667162041529082e-05
+        ],
+        "params": 5215,
+        "hidden": 27
+      },
+      "v12:FM": {
+        "mean": 1.66527801278941e-05,
+        "std": 1.5214208062517106e-05,
+        "scores": [
+          3.688366268761456e-05,
+          1.937015952080401e-07,
+          1.2880976100859698e-05
+        ],
+        "params": 5215,
+        "hidden": 27
+      },
+      "v13": {
+        "mean": 1.3201376148685995e-05,
+        "std": 1.2555101544694756e-05,
+        "scores": [
+          3.016352275153622e-05,
+          1.7497016813194932e-07,
+          9.265635526389815e-06
+        ],
+        "params": 5215,
+        "hidden": 27
+      }
+    },
+    "OOD": {
+      "Vanilla": {
+        "mean": 1.5319531361262004,
+        "std": 0.2293290940636717,
+        "scores": [
+          1.5319531361262004
+        ],
+        "params": 0,
+        "hidden": 0
+      },
+      "SinGLU": {
+        "mean": 5.897943178812663,
+        "std": 0.8500770525046415,
+        "scores": [
+          5.897943178812663
+        ],
+        "params": 0,
+        "hidden": 0
+      },
+      "v10:free": {
+        "mean": 4.963853756586711,
+        "std": 1.64714214134505,
+        "scores": [
+          4.963853756586711
+        ],
+        "params": 0,
+        "hidden": 0
+      },
+      "v12:FM": {
+        "mean": 4.501838684082031,
+        "std": 0.4557746878734239,
+        "scores": [
+          4.501838684082031
+        ],
+        "params": 0,
+        "hidden": 0
+      },
+      "v13": {
+        "mean": 6.137705167134603,
+        "std": 0.3306587182886255,
+        "scores": [
+          6.137705167134603
+        ],
+        "params": 0,
+        "hidden": 0
+      }
+    }
+  },
+  "ood": {
+    "Vanilla": {
+      "id": 0.21685118973255157,
+      "ood": 1.5319531361262004,
+      "deg": 7.064536459383043,
+      "is": 0.011753120779597635,
+      "os": 0.2293290940636717
+    },
+    "SinGLU": {
+      "id": 0.24567786852518717,
+      "ood": 5.897943178812663,
+      "deg": 24.006815160918734,
+      "is": 0.013796241102558409,
+      "os": 0.8500770525046415
+    },
+    "v10:free": {
+      "id": 0.0038988318604727588,
+      "ood": 4.963853756586711,
+      "deg": 1273.1643564605556,
+      "is": 0.0007464934858136934,
+      "os": 1.64714214134505
+    },
+    "v12:FM": {
+      "id": 0.10039328038692474,
+      "ood": 4.501838684082031,
+      "deg": 44.84203192416404,
+      "is": 0.010883034683370062,
+      "os": 0.4557746878734239
+    },
+    "v13": {
+      "id": 0.2554653137922287,
+      "ood": 6.137705167134603,
+      "deg": 24.025591091111615,
+      "is": 0.016522772441436077,
+      "os": 0.3306587182886255
+    }
+  },
+  "corr": {
+    "Complex": [
+      -0.07428092509508133,
+      -0.7854841351509094,
+      -0.6786586046218872
+    ],
+    "Nested": [
+      -0.022163555026054382,
+      -0.32929015159606934,
+      0.658381998538971
+    ],
+    "Spiral": [
+      0.09974421560764313,
+      0.3856152594089508,
+      0.5183417797088623
+    ],
+    "Checker": [
+      0.1836509257555008,
+      0.128373384475708,
+      0.2680778503417969
+    ],
+    "HiFreq": [
+      -0.7121968269348145,
+      0.2593926787376404,
+      0.15112583339214325
+    ],
+    "Memorize": [
+      0.15285518765449524,
+      0.1444067656993866,
+      0.2329363077878952
+    ]
+  }
+}