Upload add_sub_sorl_v1_abs10_K1_25K_1L3H510d/metrics.json with huggingface_hub

add_sub_sorl_v1_abs10_K1_25K_1L3H510d/metrics.json

@@ -1190,502 +1190,567 @@
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.91,
-        "n_examples": 100,
+        "full_accuracy": 0.88,
+        "digit_accuracy": 0.9828571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9851239669421488,
-            "count": 605
+            "accuracy": 0.9796610169491525,
+            "count": 295
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 95
+            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.86,
-        "n_examples": 100,
+        "full_accuracy": 0.82,
+        "digit_accuracy": 0.9714285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9852941176470589,
-            "count": 204
+            "accuracy": 0.9920634920634921,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.9822485207100592,
-            "count": 169
+            "accuracy": 0.9873417721518988,
+            "count": 79
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 31
+            "accuracy": 0.9523809523809523,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.9695945945945946,
-            "count": 296
+            "accuracy": 0.9435483870967742,
+            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.77,
-        "n_examples": 100,
+        "full_accuracy": 0.66,
+        "digit_accuracy": 0.9371428571428572,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 163
+            "accuracy": 0.9733333333333334,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.9615384615384616,
-            "count": 130
+            "accuracy": 0.9516129032258065,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.9770114942528736,
-            "count": 87
+            "accuracy": 0.9230769230769231,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.9310344827586207,
-            "count": 203
+            "accuracy": 0.8828828828828829,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.9743589743589743,
-            "count": 117
+            "accuracy": 0.9841269841269841,
+            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.53,
-        "n_examples": 100,
+        "full_accuracy": 0.58,
+        "digit_accuracy": 0.9285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9834710743801653,
-            "count": 121
+            "accuracy": 1.0,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.9669421487603306,
-            "count": 121
+            "accuracy": 0.9649122807017544,
+            "count": 57
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 49
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.8064516129032258,
-            "count": 186
+            "accuracy": 0.8076923076923077,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.9551569506726457,
-            "count": 223
+            "accuracy": 0.9727272727272728,
+            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.4,
-        "n_examples": 100,
+        "full_accuracy": 0.46,
+        "digit_accuracy": 0.8628571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 104
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.9716981132075472,
-            "count": 106
+            "accuracy": 1.0,
+            "count": 52
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 23
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.8125,
-            "count": 160
+            "accuracy": 0.7528089887640449,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.8045602605863192,
-            "count": 307
+            "accuracy": 0.8311688311688312,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.27,
-        "n_examples": 100,
+        "full_accuracy": 0.28,
+        "digit_accuracy": 0.6857142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.58,
-            "count": 100
+            "accuracy": 0.56,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.6425,
-            "count": 400
+            "accuracy": 0.56,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.61,
-        "n_examples": 100,
+        "full_accuracy": 0.64,
+        "digit_accuracy": 0.7914285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.74,
-            "count": 100
+            "accuracy": 0.76,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.766,
-            "count": 500
+            "accuracy": 0.756,
+            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.865,
+        "full_accuracy": 0.85,
+        "digit_accuracy": 0.9764285714285714,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9910514541387024,
-            "count": 447
+            "accuracy": 0.988399071925754,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.975,
-            "count": 320
+            "accuracy": 0.9810126582278481,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.9821428571428571,
-            "count": 56
+            "accuracy": 0.9743589743589743,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.9754253308128544,
-            "count": 529
+            "accuracy": 0.9625,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.9583333333333334,
-            "count": 48
+            "accuracy": 1.0,
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.84,
+        "digit_accuracy": 0.9771428571428571,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.84,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.84,
+        "digit_accuracy": 0.9657142857142857,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.9156626506024096,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 0.7058823529411765,
+            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.61,
-        "n_examples": 100,
+        "full_accuracy": 0.56,
+        "digit_accuracy": 0.9257142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 300
+            "count": 150
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8082901554404145,
-            "count": 193
+            "accuracy": 0.85,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.9158878504672897,
-            "count": 107
+            "accuracy": 0.78,
+            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.65,
-        "n_examples": 100,
+        "full_accuracy": 0.7,
+        "digit_accuracy": 0.9514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.88671875,
-            "count": 256
+            "accuracy": 0.9090909090909091,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.8888888888888888,
-            "count": 144
+            "accuracy": 0.9264705882352942,
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.51,
-        "n_examples": 100,
+        "full_accuracy": 0.6,
+        "digit_accuracy": 0.9228571428571428,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8562091503267973,
-            "count": 306
+            "accuracy": 0.9041095890410958,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.9278350515463918,
-            "count": 194
+            "accuracy": 0.875,
+            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.68,
-        "n_examples": 100,
+        "full_accuracy": 0.56,
+        "digit_accuracy": 0.9142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.9153005464480874,
-            "count": 366
+            "accuracy": 0.8994708994708994,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.9700854700854701,
-            "count": 234
+            "accuracy": 0.9009009009009009,
+            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.88,
-        "n_examples": 100,
+        "full_accuracy": 0.86,
+        "digit_accuracy": 0.98,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9866888519134775,
-            "count": 601
+            "accuracy": 0.976897689768977,
+            "count": 303
           },
           "ME": {
-            "accuracy": 0.9595959595959596,
-            "count": 99
+            "accuracy": 1.0,
+            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.78,
-        "n_examples": 100,
+        "full_accuracy": 0.72,
+        "digit_accuracy": 0.9571428571428572,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.985663082437276,
-            "count": 279
+            "accuracy": 1.0,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.9586206896551724,
-            "count": 145
+            "accuracy": 0.9722222222222222,
+            "count": 72
           },
           "ME": {
-            "accuracy": 0.9583333333333334,
-            "count": 24
+            "accuracy": 0.8888888888888888,
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.9484126984126984,
-            "count": 252
+            "accuracy": 0.907563025210084,
+            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.35,
-        "n_examples": 100,
+        "full_accuracy": 0.3,
+        "digit_accuracy": 0.8714285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9765258215962441,
-            "count": 213
+            "accuracy": 0.9732142857142857,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.9734513274336283,
-            "count": 113
+            "accuracy": 0.9056603773584906,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.9764705882352941,
-            "count": 85
+            "accuracy": 0.9574468085106383,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.6629834254143646,
-            "count": 181
+            "accuracy": 0.6470588235294118,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.9074074074074074,
-            "count": 108
+            "accuracy": 0.9056603773584906,
+            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.11,
-        "n_examples": 100,
+        "full_accuracy": 0.2,
+        "digit_accuracy": 0.8,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.994413407821229,
-            "count": 179
+            "accuracy": 0.9896907216494846,
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.970873786407767,
-            "count": 103
+            "accuracy": 0.9607843137254902,
+            "count": 51
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.5167785234899329,
-            "count": 149
+            "accuracy": 0.5405405405405406,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.6384976525821596,
-            "count": 213
+            "accuracy": 0.6732673267326733,
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.09,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.6428571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.34,
-            "count": 100
+            "accuracy": 0.26,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.4,
-            "count": 300
+            "accuracy": 0.41333333333333333,
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.0,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.44285714285714284,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.24,
-            "count": 100
+            "accuracy": 0.32,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.22,
-            "count": 400
+            "accuracy": 0.195,
+            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.705,
+        "full_accuracy": 0.765,
+        "digit_accuracy": 0.9628571428571429,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.985,
-            "count": 600
+            "accuracy": 0.9894736842105263,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.9662921348314607,
-            "count": 267
+            "accuracy": 0.9566787003610109,
+            "count": 277
           },
           "ME": {
             "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.9020501138952164,
-            "count": 439
+            "accuracy": 0.9299363057324841,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.975609756097561,
-            "count": 41
+            "accuracy": 0.9655172413793104,
+            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.38,
-        "n_examples": 100,
+        "full_accuracy": 0.48,
+        "digit_accuracy": 0.8971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.99,
-            "count": 300
+            "accuracy": 1.0,
+            "count": 150
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.7055837563451777,
-            "count": 197
+            "accuracy": 0.7623762376237624,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.6893203883495146,
-            "count": 103
+            "accuracy": 0.7551020408163265,
+            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.27,
-        "n_examples": 100,
+        "full_accuracy": 0.24,
+        "digit_accuracy": 0.82,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.6882591093117408,
-            "count": 247
+            "accuracy": 0.6942148760330579,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.6405228758169934,
-            "count": 153
+            "accuracy": 0.6708860759493671,
+            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.2,
-        "n_examples": 100,
+        "full_accuracy": 0.14,
+        "digit_accuracy": 0.7714285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.7248322147651006,
-            "count": 298
+            "accuracy": 0.6907894736842105,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.6287128712871287,
-            "count": 202
+            "accuracy": 0.6632653061224489,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.5416666666666666,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.594,
+      "digit_accuracy": 0.8919047619047619,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_eval": {
@@ -1694,502 +1759,567 @@
       "K": 1,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.96,
-        "n_examples": 100,
+        "full_accuracy": 0.94,
+        "digit_accuracy": 0.9914285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9933884297520661,
-            "count": 605
+            "accuracy": 0.9898305084745763,
+            "count": 295
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 95
+            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 0.9,
-        "n_examples": 100,
+        "digit_accuracy": 0.9857142857142858,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9852941176470589,
-            "count": 204
+            "accuracy": 0.9920634920634921,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.9881656804733728,
-            "count": 169
+            "accuracy": 0.9620253164556962,
+            "count": 79
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 31
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.9831081081081081,
-            "count": 296
+            "accuracy": 0.9919354838709677,
+            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.69,
-        "n_examples": 100,
+        "full_accuracy": 0.7,
+        "digit_accuracy": 0.9457142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9877300613496932,
-            "count": 163
+            "accuracy": 0.9866666666666667,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.9384615384615385,
-            "count": 130
+            "accuracy": 0.9193548387096774,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.9310344827586207,
-            "count": 87
+            "accuracy": 0.8974358974358975,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.9064039408866995,
-            "count": 203
+            "accuracy": 0.918918918918919,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.9914529914529915,
-            "count": 117
+            "accuracy": 1.0,
+            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.57,
-        "n_examples": 100,
+        "full_accuracy": 0.6,
+        "digit_accuracy": 0.9285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 121
+            "accuracy": 0.9833333333333333,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.9752066115702479,
-            "count": 121
+            "accuracy": 0.9824561403508771,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.9795918367346939,
-            "count": 49
+            "accuracy": 0.9473684210526315,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.8279569892473119,
-            "count": 186
+            "accuracy": 0.8269230769230769,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.8923766816143498,
-            "count": 223
+            "accuracy": 0.9636363636363636,
+            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.58,
-        "n_examples": 100,
+        "full_accuracy": 0.68,
+        "digit_accuracy": 0.8971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 104
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.9905660377358491,
-            "count": 106
+            "accuracy": 1.0,
+            "count": 52
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 23
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.825,
-            "count": 160
+            "accuracy": 0.8651685393258427,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.8306188925081434,
-            "count": 307
+            "accuracy": 0.8441558441558441,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.37,
-        "n_examples": 100,
+        "full_accuracy": 0.4,
+        "digit_accuracy": 0.7514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.54,
-            "count": 100
+            "accuracy": 0.62,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.6425,
-            "count": 400
+            "accuracy": 0.66,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.61,
-        "n_examples": 100,
+        "full_accuracy": 0.48,
+        "digit_accuracy": 0.76,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.69,
-            "count": 100
+            "accuracy": 0.64,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.77,
-            "count": 500
+            "accuracy": 0.736,
+            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.925,
+        "full_accuracy": 0.9,
+        "digit_accuracy": 0.9842857142857143,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9932885906040269,
-            "count": 447
+            "accuracy": 0.9953596287703016,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.9875,
-            "count": 320
+            "accuracy": 0.990506329113924,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.9821428571428571,
-            "count": 56
+            "accuracy": 0.9743589743589743,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.9848771266540642,
-            "count": 529
+            "accuracy": 0.9732142857142857,
+            "count": 560
           },
           "US": {
+            "accuracy": 0.9814814814814815,
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.96,
+        "digit_accuracy": 0.9942857142857143,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
             "accuracy": 1.0,
-            "count": 48
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.96,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.9,
+        "digit_accuracy": 0.98,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.963855421686747,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 0.7647058823529411,
+            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.62,
-        "n_examples": 100,
+        "full_accuracy": 0.58,
+        "digit_accuracy": 0.9142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 300
+            "count": 150
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8186528497409327,
-            "count": 193
+            "accuracy": 0.85,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.8785046728971962,
-            "count": 107
+            "accuracy": 0.7,
+            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.61,
-        "n_examples": 100,
+        "full_accuracy": 0.6,
+        "digit_accuracy": 0.9285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.87890625,
-            "count": 256
+            "accuracy": 0.8712121212121212,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.8611111111111112,
-            "count": 144
+            "accuracy": 0.8823529411764706,
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.59,
-        "n_examples": 100,
+        "full_accuracy": 0.62,
+        "digit_accuracy": 0.9085714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8823529411764706,
-            "count": 306
+            "accuracy": 0.8698630136986302,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.8969072164948454,
-            "count": 194
+            "accuracy": 0.875,
+            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.69,
-        "n_examples": 100,
+        "full_accuracy": 0.62,
+        "digit_accuracy": 0.9171428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.912568306010929,
-            "count": 366
+            "accuracy": 0.8994708994708994,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.9444444444444444,
-            "count": 234
+            "accuracy": 0.9099099099099099,
+            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.89,
-        "n_examples": 100,
+        "full_accuracy": 0.76,
+        "digit_accuracy": 0.96,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9850249584026622,
-            "count": 601
+            "accuracy": 0.9570957095709571,
+            "count": 303
           },
           "ME": {
-            "accuracy": 0.9696969696969697,
-            "count": 99
+            "accuracy": 0.9787234042553191,
+            "count": 47
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 0.82,
-        "n_examples": 100,
+        "digit_accuracy": 0.9657142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.982078853046595,
-            "count": 279
+            "accuracy": 0.9858156028368794,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.9793103448275862,
-            "count": 145
+            "accuracy": 0.9861111111111112,
+            "count": 72
           },
           "ME": {
-            "accuracy": 0.9583333333333334,
-            "count": 24
+            "accuracy": 0.8888888888888888,
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.9603174603174603,
-            "count": 252
+            "accuracy": 0.9411764705882353,
+            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.33,
-        "n_examples": 100,
+        "full_accuracy": 0.32,
+        "digit_accuracy": 0.8828571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.971830985915493,
-            "count": 213
+            "accuracy": 0.9464285714285714,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.9380530973451328,
-            "count": 113
+            "accuracy": 0.9433962264150944,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.9764705882352941,
-            "count": 85
+            "accuracy": 0.9787234042553191,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.6464088397790055,
-            "count": 181
+            "accuracy": 0.6705882352941176,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.9444444444444444,
-            "count": 108
+            "accuracy": 0.9433962264150944,
+            "count": 53
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 0.22,
-        "n_examples": 100,
+        "digit_accuracy": 0.7885714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9832402234636871,
-            "count": 179
+            "accuracy": 0.9896907216494846,
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.9805825242718447,
-            "count": 103
+            "accuracy": 0.9607843137254902,
+            "count": 51
           },
           "ME": {
-            "accuracy": 1.0,
-            "count": 56
+            "accuracy": 0.8888888888888888,
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.5704697986577181,
-            "count": 149
+            "accuracy": 0.5405405405405406,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.7136150234741784,
-            "count": 213
+            "accuracy": 0.6633663366336634,
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.2,
-        "n_examples": 100,
+        "full_accuracy": 0.08,
+        "digit_accuracy": 0.6914285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.99,
-            "count": 200
+            "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.49,
-            "count": 100
+            "accuracy": 0.44,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.49,
-            "count": 300
+            "accuracy": 0.4666666666666667,
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.1,
-        "n_examples": 100,
+        "full_accuracy": 0.24,
+        "digit_accuracy": 0.6028571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.4,
-            "count": 100
+            "accuracy": 0.56,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.33,
-            "count": 400
+            "accuracy": 0.415,
+            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.735,
+        "full_accuracy": 0.825,
+        "digit_accuracy": 0.9735714285714285,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9766666666666667,
-            "count": 600
+            "accuracy": 0.9894736842105263,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.9588014981273408,
-            "count": 267
+            "accuracy": 0.9819494584837545,
+            "count": 277
           },
           "ME": {
             "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.9225512528473804,
-            "count": 439
+            "accuracy": 0.9490445859872612,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.975609756097561,
-            "count": 41
+            "accuracy": 0.9310344827586207,
+            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.44,
-        "n_examples": 100,
+        "full_accuracy": 0.46,
+        "digit_accuracy": 0.9,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.98,
-            "count": 300
+            "accuracy": 0.9933333333333333,
+            "count": 150
           },
           "MB": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.7360406091370558,
-            "count": 197
+            "accuracy": 0.7722772277227723,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.7766990291262136,
-            "count": 103
+            "accuracy": 0.7755102040816326,
+            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.39,
-        "n_examples": 100,
+        "full_accuracy": 0.32,
+        "digit_accuracy": 0.8285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 0.98,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.7530364372469636,
-            "count": 247
+            "accuracy": 0.6859504132231405,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.7058823529411765,
-            "count": 153
+            "accuracy": 0.7341772151898734,
+            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.27,
-        "n_examples": 100,
+        "full_accuracy": 0.18,
+        "digit_accuracy": 0.7657142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.7583892617449665,
-            "count": 298
+            "accuracy": 0.6578947368421053,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.7128712871287128,
-            "count": 202
+            "accuracy": 0.6938775510204082,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.5895833333333333,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.642,
+      "digit_accuracy": 0.9039047619047619,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_overall_accuracy": 0.5895833333333333,