Upload add_sub_sorl_v1_abs30_K1_100K_1L2H256d/metrics.json with huggingface_hub

add_sub_sorl_v1_abs30_K1_100K_1L2H256d/metrics.json

@@ -5030,502 +5030,567 @@
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.73,
-        "n_examples": 100,
+        "full_accuracy": 0.72,
+        "digit_accuracy": 0.9514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9520661157024793,
-            "count": 605
+            "accuracy": 0.9457627118644067,
+            "count": 295
           },
           "SS": {
-            "accuracy": 0.9789473684210527,
-            "count": 95
+            "accuracy": 0.9818181818181818,
+            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.59,
-        "n_examples": 100,
+        "full_accuracy": 0.6,
+        "digit_accuracy": 0.9314285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9362745098039216,
-            "count": 204
+            "accuracy": 0.9444444444444444,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.9349112426035503,
-            "count": 169
+            "accuracy": 0.9620253164556962,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.9354838709677419,
-            "count": 31
+            "accuracy": 1.0,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.918918918918919,
-            "count": 296
+            "accuracy": 0.8870967741935484,
+            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.3,
-        "n_examples": 100,
+        "full_accuracy": 0.2,
+        "digit_accuracy": 0.8085714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9386503067484663,
-            "count": 163
+            "accuracy": 0.9066666666666666,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.8692307692307693,
-            "count": 130
+            "accuracy": 0.8387096774193549,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.8735632183908046,
-            "count": 87
+            "accuracy": 0.6410256410256411,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.7192118226600985,
-            "count": 203
+            "accuracy": 0.7117117117117117,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.9401709401709402,
-            "count": 117
+            "accuracy": 0.9365079365079365,
+            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.21,
-        "n_examples": 100,
+        "full_accuracy": 0.24,
+        "digit_accuracy": 0.8085714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9669421487603306,
-            "count": 121
+            "accuracy": 0.95,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.8925619834710744,
-            "count": 121
+            "accuracy": 0.8596491228070176,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.8775510204081632,
-            "count": 49
+            "accuracy": 1.0,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.6720430107526881,
-            "count": 186
+            "accuracy": 0.7115384615384616,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.7937219730941704,
-            "count": 223
+            "accuracy": 0.7636363636363637,
+            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.2,
-        "n_examples": 100,
+        "full_accuracy": 0.24,
+        "digit_accuracy": 0.7657142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9519230769230769,
-            "count": 104
+            "accuracy": 1.0,
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.8773584905660378,
-            "count": 106
+            "accuracy": 0.9038461538461539,
+            "count": 52
           },
           "SS": {
-            "accuracy": 0.9565217391304348,
-            "count": 23
+            "accuracy": 1.0,
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.7375,
-            "count": 160
+            "accuracy": 0.7078651685393258,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.6416938110749185,
-            "count": 307
+            "accuracy": 0.6688311688311688,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.07,
-        "n_examples": 100,
+        "full_accuracy": 0.08,
+        "digit_accuracy": 0.5228571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 0.98,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.29,
-            "count": 100
+            "accuracy": 0.34,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.245,
-            "count": 400
+            "accuracy": 0.335,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.28,
-        "n_examples": 100,
+        "full_accuracy": 0.24,
+        "digit_accuracy": 0.52,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.57,
-            "count": 100
+            "accuracy": 0.52,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.482,
-            "count": 500
+            "accuracy": 0.424,
+            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.575,
+        "full_accuracy": 0.5,
+        "digit_accuracy": 0.9157142857142857,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9686800894854586,
-            "count": 447
+            "accuracy": 0.9675174013921114,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.925,
-            "count": 320
+            "accuracy": 0.9240506329113924,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.8928571428571429,
-            "count": 56
+            "accuracy": 0.9743589743589743,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.8960302457466919,
-            "count": 529
+            "accuracy": 0.8732142857142857,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.7708333333333334,
-            "count": 48
+            "accuracy": 0.8518518518518519,
+            "count": 54
           }
         }
       },
-      "add_C3": {
-        "full_accuracy": 0.37,
-        "n_examples": 100,
+      "add_C1": {
+        "full_accuracy": 0.68,
+        "digit_accuracy": 0.9457142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9966666666666667,
-            "count": 300
+            "accuracy": 0.992,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 0.94,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.72,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.6,
+        "digit_accuracy": 0.9342857142857143,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.995,
+            "count": 200
           },
           "SC": {
             "accuracy": 0.98,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.7098445595854922,
-            "count": 193
+            "accuracy": 0.7831325301204819,
+            "count": 83
           },
           "US": {
-            "accuracy": 0.6635514018691588,
-            "count": 107
+            "accuracy": 0.8235294117647058,
+            "count": 17
           }
         }
       },
-      "add_C4": {
-        "full_accuracy": 0.37,
-        "n_examples": 100,
+      "add_C3": {
+        "full_accuracy": 0.3,
+        "digit_accuracy": 0.8685714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.995,
-            "count": 200
+            "accuracy": 0.9933333333333333,
+            "count": 150
           },
           "SC": {
-            "accuracy": 0.99,
+            "accuracy": 0.96,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.7,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.74,
+            "count": 50
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.38,
+        "digit_accuracy": 0.86,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
             "count": 100
           },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
           "UC": {
-            "accuracy": 0.7421875,
-            "count": 256
+            "accuracy": 0.75,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.7083333333333334,
-            "count": 144
+            "accuracy": 0.7647058823529411,
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.34,
-        "n_examples": 100,
+        "full_accuracy": 0.42,
+        "digit_accuracy": 0.8285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.7483660130718954,
-            "count": 306
+            "accuracy": 0.7671232876712328,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.7835051546391752,
-            "count": 194
+            "accuracy": 0.75,
+            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.39,
-        "n_examples": 100,
+        "full_accuracy": 0.36,
+        "digit_accuracy": 0.8714285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.825136612021858,
-            "count": 366
+            "accuracy": 0.8253968253968254,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.8760683760683761,
-            "count": 234
+            "accuracy": 0.8918918918918919,
+            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.66,
-        "n_examples": 100,
+        "full_accuracy": 0.6,
+        "digit_accuracy": 0.9257142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9384359400998337,
-            "count": 601
+            "accuracy": 0.9174917491749175,
+            "count": 303
           },
           "ME": {
-            "accuracy": 0.9696969696969697,
-            "count": 99
+            "accuracy": 0.9787234042553191,
+            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.45,
-        "n_examples": 100,
+        "full_accuracy": 0.42,
+        "digit_accuracy": 0.8914285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.956989247311828,
-            "count": 279
+            "accuracy": 0.9645390070921985,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.9517241379310345,
-            "count": 145
+            "accuracy": 0.8611111111111112,
+            "count": 72
           },
           "ME": {
-            "accuracy": 0.875,
-            "count": 24
+            "accuracy": 0.9444444444444444,
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.8015873015873016,
-            "count": 252
+            "accuracy": 0.8151260504201681,
+            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.18,
-        "n_examples": 100,
+        "full_accuracy": 0.16,
+        "digit_accuracy": 0.8085714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9624413145539906,
-            "count": 213
+            "accuracy": 0.9464285714285714,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.8761061946902655,
-            "count": 113
+            "accuracy": 0.6981132075471698,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.9411764705882353,
-            "count": 85
+            "accuracy": 0.9361702127659575,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.6353591160220995,
-            "count": 181
+            "accuracy": 0.6352941176470588,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.7685185185185185,
-            "count": 108
+            "accuracy": 0.7924528301886793,
+            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.07,
-        "n_examples": 100,
+        "full_accuracy": 0.06,
+        "digit_accuracy": 0.7285714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9832402234636871,
-            "count": 179
+            "accuracy": 0.979381443298969,
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.8737864077669902,
-            "count": 103
+            "accuracy": 0.9215686274509803,
+            "count": 51
           },
           "ME": {
-            "accuracy": 0.9821428571428571,
-            "count": 56
+            "accuracy": 0.9629629629629629,
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.47651006711409394,
-            "count": 149
+            "accuracy": 0.5,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.49295774647887325,
-            "count": 213
+            "accuracy": 0.49504950495049505,
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.07,
-        "n_examples": 100,
+        "full_accuracy": 0.04,
+        "digit_accuracy": 0.58,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 0.96,
-            "count": 200
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.96,
-            "count": 100
+            "accuracy": 0.98,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.29,
-            "count": 100
+            "accuracy": 0.4,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.30333333333333334,
-            "count": 300
+            "accuracy": 0.25333333333333335,
+            "count": 150
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.04,
-        "n_examples": 100,
+        "digit_accuracy": 0.4514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.37,
-            "count": 100
+            "accuracy": 0.34,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.2275,
-            "count": 400
+            "accuracy": 0.205,
+            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.465,
+        "full_accuracy": 0.415,
+        "digit_accuracy": 0.8857142857142857,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9433333333333334,
-            "count": 600
+            "accuracy": 0.9631578947368421,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.9213483146067416,
-            "count": 267
+            "accuracy": 0.8772563176895307,
+            "count": 277
           },
           "ME": {
-            "accuracy": 0.9245283018867925,
+            "accuracy": 0.9433962264150944,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.806378132118451,
-            "count": 439
+            "accuracy": 0.7983014861995754,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.7804878048780488,
-            "count": 41
+            "accuracy": 0.7586206896551724,
+            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.21,
-        "n_examples": 100,
+        "full_accuracy": 0.2,
+        "digit_accuracy": 0.8171428571428572,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9433333333333334,
-            "count": 300
+            "accuracy": 0.9466666666666667,
+            "count": 150
           },
           "MB": {
-            "accuracy": 0.95,
-            "count": 100
+            "accuracy": 0.96,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.5939086294416244,
-            "count": 197
+            "accuracy": 0.7029702970297029,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.6019417475728155,
-            "count": 103
+            "accuracy": 0.5102040816326531,
+            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.17,
-        "n_examples": 100,
+        "full_accuracy": 0.2,
+        "digit_accuracy": 0.7142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.925,
-            "count": 200
+            "accuracy": 0.95,
+            "count": 100
           },
           "MB": {
             "accuracy": 0.98,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.5951417004048583,
-            "count": 247
+            "accuracy": 0.5950413223140496,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.5947712418300654,
-            "count": 153
+            "accuracy": 0.43037974683544306,
+            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.1,
-        "n_examples": 100,
+        "full_accuracy": 0.16,
+        "digit_accuracy": 0.6914285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.5838926174496645,
-            "count": 298
+            "accuracy": 0.625,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.49504950495049505,
-            "count": 202
+            "accuracy": 0.47959183673469385,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.3279166666666667,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.35333333333333333,
+      "digit_accuracy": 0.8141904761904762,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_eval": {
@@ -5534,502 +5599,567 @@
       "K": 1,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.91,
-        "n_examples": 100,
+        "full_accuracy": 0.86,
+        "digit_accuracy": 0.98,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9867768595041322,
-            "count": 605
+            "accuracy": 0.9796610169491525,
+            "count": 295
           },
           "SS": {
-            "accuracy": 0.9894736842105263,
-            "count": 95
+            "accuracy": 0.9818181818181818,
+            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.64,
-        "n_examples": 100,
+        "full_accuracy": 0.74,
+        "digit_accuracy": 0.96,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9705882352941176,
-            "count": 204
+            "accuracy": 0.9761904761904762,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.9526627218934911,
-            "count": 169
+            "accuracy": 0.9873417721518988,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.967741935483871,
-            "count": 31
+            "accuracy": 1.0,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.918918918918919,
-            "count": 296
+            "accuracy": 0.9193548387096774,
+            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.55,
-        "n_examples": 100,
+        "full_accuracy": 0.5,
+        "digit_accuracy": 0.9142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9877300613496932,
-            "count": 163
+            "accuracy": 1.0,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.9230769230769231,
-            "count": 130
+            "accuracy": 0.9354838709677419,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.9310344827586207,
-            "count": 87
+            "accuracy": 0.9487179487179487,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.8374384236453202,
-            "count": 203
+            "accuracy": 0.8108108108108109,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.9572649572649573,
-            "count": 117
+            "accuracy": 0.9523809523809523,
+            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.47,
-        "n_examples": 100,
+        "full_accuracy": 0.54,
+        "digit_accuracy": 0.9,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 121
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.9752066115702479,
-            "count": 121
+            "accuracy": 0.9824561403508771,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.9183673469387755,
-            "count": 49
+            "accuracy": 1.0,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.7634408602150538,
-            "count": 186
+            "accuracy": 0.7980769230769231,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.8565022421524664,
-            "count": 223
+            "accuracy": 0.8818181818181818,
+            "count": 110
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.4,
-        "n_examples": 100,
+        "digit_accuracy": 0.8485714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9903846153846154,
-            "count": 104
+            "accuracy": 1.0,
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.9339622641509434,
-            "count": 106
+            "accuracy": 0.9615384615384616,
+            "count": 52
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 23
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.7125,
-            "count": 160
+            "accuracy": 0.7528089887640449,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.7850162866449512,
-            "count": 307
+            "accuracy": 0.8116883116883117,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.1,
-        "n_examples": 100,
+        "full_accuracy": 0.06,
+        "digit_accuracy": 0.58,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
-            "accuracy": 1.0,
-            "count": 100
+            "accuracy": 0.98,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.4,
-            "count": 100
+            "accuracy": 0.26,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.525,
-            "count": 400
+            "accuracy": 0.455,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.19,
-        "n_examples": 100,
+        "full_accuracy": 0.14,
+        "digit_accuracy": 0.5285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.35,
-            "count": 100
+            "accuracy": 0.24,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.534,
-            "count": 500
+            "accuracy": 0.492,
+            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.73,
+        "full_accuracy": 0.71,
+        "digit_accuracy": 0.9528571428571428,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9753914988814317,
-            "count": 447
+            "accuracy": 0.9791183294663574,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.96875,
-            "count": 320
+            "accuracy": 0.9430379746835443,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.8571428571428571,
-            "count": 56
+            "accuracy": 0.9743589743589743,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.9376181474480151,
-            "count": 529
+            "accuracy": 0.9357142857142857,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.8541666666666666,
-            "count": 48
+            "accuracy": 0.9629629629629629,
+            "count": 54
           }
         }
       },
-      "add_C3": {
-        "full_accuracy": 0.43,
-        "n_examples": 100,
+      "add_C1": {
+        "full_accuracy": 0.78,
+        "digit_accuracy": 0.9685714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9966666666666667,
-            "count": 300
+            "accuracy": 1.0,
+            "count": 250
           },
           "SC": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.7305699481865285,
-            "count": 193
+            "accuracy": 0.78,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.58,
+        "digit_accuracy": 0.9228571428571428,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.985,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.96,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.7831325301204819,
+            "count": 83
           },
           "US": {
-            "accuracy": 0.7476635514018691,
-            "count": 107
+            "accuracy": 0.7647058823529411,
+            "count": 17
           }
         }
       },
-      "add_C4": {
-        "full_accuracy": 0.52,
-        "n_examples": 100,
+      "add_C3": {
+        "full_accuracy": 0.5,
+        "digit_accuracy": 0.8942857142857142,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 150
           },
           "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.75,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.76,
+            "count": 50
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.54,
+        "digit_accuracy": 0.9057142857142857,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
             "accuracy": 0.99,
             "count": 100
           },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
           "UC": {
-            "accuracy": 0.83203125,
-            "count": 256
+            "accuracy": 0.8636363636363636,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.8402777777777778,
-            "count": 144
+            "accuracy": 0.7941176470588235,
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.41,
-        "n_examples": 100,
+        "full_accuracy": 0.54,
+        "digit_accuracy": 0.88,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.7973856209150327,
-            "count": 306
+            "accuracy": 0.815068493150685,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.8247422680412371,
-            "count": 194
+            "accuracy": 0.8557692307692307,
+            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.49,
-        "n_examples": 100,
+        "full_accuracy": 0.48,
+        "digit_accuracy": 0.8771428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8797814207650273,
-            "count": 366
+            "accuracy": 0.8518518518518519,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.8888888888888888,
-            "count": 234
+            "accuracy": 0.8648648648648649,
+            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.93,
-        "n_examples": 100,
+        "full_accuracy": 0.88,
+        "digit_accuracy": 0.98,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9883527454242929,
-            "count": 601
+            "accuracy": 0.976897689768977,
+            "count": 303
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 99
+            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.63,
-        "n_examples": 100,
+        "full_accuracy": 0.56,
+        "digit_accuracy": 0.9257142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.985663082437276,
-            "count": 279
+            "accuracy": 0.9645390070921985,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.9586206896551724,
-            "count": 145
+            "accuracy": 0.9444444444444444,
+            "count": 72
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 24
+            "count": 18
           },
           "UB": {
             "accuracy": 0.8571428571428571,
-            "count": 252
+            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.29,
-        "n_examples": 100,
+        "full_accuracy": 0.3,
+        "digit_accuracy": 0.8542857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9671361502347418,
-            "count": 213
+            "accuracy": 0.9910714285714286,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.9203539823008849,
-            "count": 113
+            "accuracy": 0.8301886792452831,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.9764705882352941,
-            "count": 85
+            "accuracy": 0.9787234042553191,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.6243093922651933,
-            "count": 181
+            "accuracy": 0.6,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.8333333333333334,
-            "count": 108
+            "accuracy": 0.8867924528301887,
+            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.25,
-        "n_examples": 100,
+        "full_accuracy": 0.24,
+        "digit_accuracy": 0.7971428571428572,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.994413407821229,
-            "count": 179
+            "accuracy": 0.9896907216494846,
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.9223300970873787,
-            "count": 103
+            "accuracy": 0.9607843137254902,
+            "count": 51
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 56
+            "accuracy": 0.9629629629629629,
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.5436241610738255,
-            "count": 149
+            "accuracy": 0.5405405405405406,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.6384976525821596,
-            "count": 213
+            "accuracy": 0.6732673267326733,
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.07,
-        "n_examples": 100,
+        "full_accuracy": 0.1,
+        "digit_accuracy": 0.6257142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.99,
-            "count": 200
+            "accuracy": 0.98,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.95,
-            "count": 100
+            "accuracy": 0.96,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.41,
-            "count": 100
+            "accuracy": 0.36,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.34,
-            "count": 300
+            "accuracy": 0.36666666666666664,
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.05,
-        "n_examples": 100,
+        "full_accuracy": 0.06,
+        "digit_accuracy": 0.48857142857142855,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.83,
-            "count": 100
+            "accuracy": 0.88,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.34,
-            "count": 100
+            "accuracy": 0.3,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.2375,
-            "count": 400
+            "accuracy": 0.31,
+            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.585,
+        "full_accuracy": 0.59,
+        "digit_accuracy": 0.9321428571428572,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9666666666666667,
-            "count": 600
+            "accuracy": 0.9754385964912281,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.9700374531835206,
-            "count": 267
+            "accuracy": 0.9638989169675091,
+            "count": 277
           },
           "ME": {
-            "accuracy": 0.9245283018867925,
+            "accuracy": 0.9811320754716981,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.8405466970387244,
-            "count": 439
+            "accuracy": 0.8598726114649682,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.8048780487804879,
-            "count": 41
+            "accuracy": 0.8620689655172413,
+            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.34,
-        "n_examples": 100,
+        "full_accuracy": 0.26,
+        "digit_accuracy": 0.8142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.98,
-            "count": 300
+            "accuracy": 0.9666666666666667,
+            "count": 150
           },
           "MB": {
-            "accuracy": 0.96,
-            "count": 100
+            "accuracy": 0.98,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.6598984771573604,
-            "count": 197
+            "accuracy": 0.6237623762376238,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.6699029126213593,
-            "count": 103
+            "accuracy": 0.5714285714285714,
+            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.19,
-        "n_examples": 100,
+        "full_accuracy": 0.28,
+        "digit_accuracy": 0.7857142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.98,
-            "count": 200
-          },
-          "MB": {
             "accuracy": 0.99,
             "count": 100
           },
+          "MB": {
+            "accuracy": 0.98,
+            "count": 50
+          },
           "UB": {
-            "accuracy": 0.631578947368421,
-            "count": 247
+            "accuracy": 0.6694214876033058,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.5555555555555556,
-            "count": 153
+            "accuracy": 0.5822784810126582,
+            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.17,
-        "n_examples": 100,
+        "full_accuracy": 0.14,
+        "digit_accuracy": 0.7571428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.6610738255033557,
-            "count": 298
+            "accuracy": 0.7171052631578947,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.5544554455445545,
-            "count": 202
+            "accuracy": 0.5714285714285714,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.44375,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.4886666666666667,
+      "digit_accuracy": 0.8573333333333333,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_overall_accuracy": 0.44375,