Upload add_sub_baseline_10K_1L2H256d/metrics.json with huggingface_hub

add_sub_baseline_10K_1L2H256d/metrics.json

@@ -330,502 +330,567 @@
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3457142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.2066115702479339,
-            "count": 605
+            "accuracy": 0.22372881355932203,
+            "count": 295
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 95
+            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.24571428571428572,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.2647058823529412,
-            "count": 204
+            "accuracy": 0.29365079365079366,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.1242603550295858,
-            "count": 169
+            "accuracy": 0.08860759493670886,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.8709677419354839,
-            "count": 31
+            "accuracy": 0.9047619047619048,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.2533783783783784,
-            "count": 296
+            "accuracy": 0.18548387096774194,
+            "count": 124
           }
         }
       },
       "add_S2": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.37142857142857144,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.3987730061349693,
-            "count": 163
+            "accuracy": 0.4,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.09230769230769231,
-            "count": 130
+            "accuracy": 0.12903225806451613,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.5517241379310345,
-            "count": 87
+            "accuracy": 0.5128205128205128,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.3842364532019704,
-            "count": 203
+            "accuracy": 0.36936936936936937,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.4700854700854701,
-            "count": 117
+            "accuracy": 0.49206349206349204,
+            "count": 63
           }
         }
       },
       "add_S3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.37142857142857144,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.49586776859504134,
-            "count": 121
+            "accuracy": 0.5,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.024793388429752067,
-            "count": 121
+            "accuracy": 0.10526315789473684,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.8571428571428571,
-            "count": 49
+            "accuracy": 0.6842105263157895,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.3118279569892473,
-            "count": 186
+            "accuracy": 0.3557692307692308,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.3632286995515695,
-            "count": 223
+            "accuracy": 0.4,
+            "count": 110
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.32,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.4807692307692308,
-            "count": 104
+            "accuracy": 0.4791666666666667,
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.0660377358490566,
-            "count": 106
+            "accuracy": 0.057692307692307696,
+            "count": 52
           },
           "SS": {
-            "accuracy": 0.6956521739130435,
-            "count": 23
+            "accuracy": 0.8571428571428571,
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.38125,
-            "count": 160
+            "accuracy": 0.3146067415730337,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.30293159609120524,
-            "count": 307
+            "accuracy": 0.33766233766233766,
+            "count": 154
           }
         }
       },
       "add_S5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.07714285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.46,
-            "count": 100
+            "accuracy": 0.5,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.02,
-            "count": 100
+            "accuracy": 0.0,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.15,
-            "count": 100
+            "accuracy": 0.04,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.0525,
-            "count": 400
+            "accuracy": 0.0,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.04,
-        "n_examples": 100,
+        "full_accuracy": 0.12,
+        "digit_accuracy": 0.3485714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.05,
-            "count": 100
+            "accuracy": 0.12,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.64,
-            "count": 100
+            "accuracy": 0.52,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.562,
-            "count": 500
+            "accuracy": 0.36,
+            "count": 250
           }
         }
       },
       "add_random": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.24357142857142858,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.2684563758389262,
-            "count": 447
+            "accuracy": 0.2505800464037123,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.078125,
-            "count": 320
+            "accuracy": 0.10126582278481013,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.8571428571428571,
-            "count": 56
+            "accuracy": 0.8974358974358975,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.2646502835538752,
-            "count": 529
+            "accuracy": 0.2571428571428571,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.3958333333333333,
-            "count": 48
+            "accuracy": 0.4074074074074074,
+            "count": 54
           }
         }
       },
-      "add_C3": {
+      "add_C1": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.15714285714285714,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.184,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 0.06,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.12,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.16285714285714287,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.24333333333333335,
-            "count": 300
+            "accuracy": 0.21,
+            "count": 200
           },
           "SC": {
             "accuracy": 0.02,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.11917098445595854,
-            "count": 193
+            "accuracy": 0.13253012048192772,
+            "count": 83
           },
           "US": {
-            "accuracy": 0.07476635514018691,
-            "count": 107
+            "accuracy": 0.17647058823529413,
+            "count": 17
           }
         }
       },
-      "add_C4": {
+      "add_C3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.16857142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.36,
-            "count": 200
+            "accuracy": 0.26,
+            "count": 150
           },
           "SC": {
-            "accuracy": 0.03,
+            "accuracy": 0.04,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.13,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.1,
+            "count": 50
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.18857142857142858,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.39,
             "count": 100
           },
+          "SC": {
+            "accuracy": 0.0,
+            "count": 50
+          },
           "UC": {
-            "accuracy": 0.10546875,
-            "count": 256
+            "accuracy": 0.09090909090909091,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.1736111111111111,
-            "count": 144
+            "accuracy": 0.22058823529411764,
+            "count": 68
           }
         }
       },
       "add_C5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.18857142857142858,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.53,
-            "count": 100
+            "accuracy": 0.44,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.06,
-            "count": 100
+            "accuracy": 0.04,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.1437908496732026,
-            "count": 306
+            "accuracy": 0.1506849315068493,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.2268041237113402,
-            "count": 194
+            "accuracy": 0.19230769230769232,
+            "count": 104
           }
         }
       },
       "add_C6": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.33714285714285713,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.07,
-            "count": 100
+            "accuracy": 0.06,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.2923497267759563,
-            "count": 366
+            "accuracy": 0.2962962962962963,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.6068376068376068,
-            "count": 234
+            "accuracy": 0.5315315315315315,
+            "count": 111
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3057142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.20465890183028287,
-            "count": 601
+            "accuracy": 0.19801980198019803,
+            "count": 303
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 99
+            "count": 47
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.26285714285714284,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.3835125448028674,
-            "count": 279
+            "accuracy": 0.3900709219858156,
+            "count": 141
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 145
+            "count": 72
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 24
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.09523809523809523,
-            "count": 252
+            "accuracy": 0.15966386554621848,
+            "count": 119
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.38285714285714284,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6150234741784038,
-            "count": 213
+            "accuracy": 0.6428571428571429,
+            "count": 112
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 113
+            "count": 53
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 85
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.16574585635359115,
-            "count": 181
+            "accuracy": 0.17647058823529413,
+            "count": 85
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 108
+            "count": 53
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.28,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.7597765363128491,
-            "count": 179
+            "accuracy": 0.6494845360824743,
+            "count": 97
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 103
+            "count": 51
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.12080536912751678,
-            "count": 149
+            "accuracy": 0.10810810810810811,
+            "count": 74
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 213
+            "count": 101
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.21142857142857144,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 0.5,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.3,
-            "count": 100
+            "accuracy": 0.48,
+            "count": 50
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 300
+            "count": 150
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.18285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.31,
-            "count": 100
+            "accuracy": 0.28,
+            "count": 50
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 400
+            "count": 200
           }
         }
       },
       "sub_random": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.2307142857142857,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.3616666666666667,
-            "count": 600
+            "accuracy": 0.37719298245614036,
+            "count": 570
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 267
+            "count": 277
           },
           "ME": {
             "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.12072892938496584,
-            "count": 439
+            "accuracy": 0.11677282377919321,
+            "count": 471
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 41
+            "count": 29
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.18857142857142858,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 0.3333333333333333,
-            "count": 300
+            "count": 150
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.17766497461928935,
-            "count": 197
+            "accuracy": 0.15841584158415842,
+            "count": 101
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 103
+            "count": 49
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.17714285714285713,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 0.5,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.145748987854251,
-            "count": 247
+            "accuracy": 0.09917355371900827,
+            "count": 121
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 153
+            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.18,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.11073825503355705,
-            "count": 298
+            "accuracy": 0.08552631578947369,
+            "count": 152
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 202
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.0016666666666666668,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.004,
+      "digit_accuracy": 0.24485714285714286,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   }
 }