Upload add_sub_sorl_v1_abs30_50K/metrics.json with huggingface_hub

add_sub_sorl_v1_abs30_50K/metrics.json

@@ -2470,502 +2470,567 @@
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.95,
-        "n_examples": 100,
+        "full_accuracy": 0.94,
+        "digit_accuracy": 0.9885714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9933884297520661,
-            "count": 605
+            "accuracy": 0.9932203389830508,
+            "count": 295
           },
           "SS": {
-            "accuracy": 0.9789473684210527,
-            "count": 95
+            "accuracy": 0.9636363636363636,
+            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.87,
-        "n_examples": 100,
+        "full_accuracy": 0.84,
+        "digit_accuracy": 0.9771428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9754901960784313,
-            "count": 204
+            "accuracy": 0.9761904761904762,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.9881656804733728,
-            "count": 169
+            "accuracy": 0.9746835443037974,
+            "count": 79
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 31
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.9763513513513513,
-            "count": 296
+            "accuracy": 0.9758064516129032,
+            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.65,
-        "n_examples": 100,
+        "full_accuracy": 0.5,
+        "digit_accuracy": 0.9,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9754601226993865,
-            "count": 163
+            "accuracy": 0.9733333333333334,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.9615384615384616,
-            "count": 130
+            "accuracy": 0.8870967741935484,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.9770114942528736,
-            "count": 87
+            "accuracy": 0.8461538461538461,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.8719211822660099,
-            "count": 203
+            "accuracy": 0.8468468468468469,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.9743589743589743,
-            "count": 117
+            "accuracy": 0.9523809523809523,
+            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.6,
-        "n_examples": 100,
+        "full_accuracy": 0.54,
+        "digit_accuracy": 0.8971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 1.0,
-            "count": 121
+            "accuracy": 0.9833333333333333,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.9834710743801653,
-            "count": 121
+            "accuracy": 1.0,
+            "count": 57
           },
           "SS": {
-            "accuracy": 1.0,
-            "count": 49
+            "accuracy": 0.9473684210526315,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.8387096774193549,
-            "count": 186
+            "accuracy": 0.8365384615384616,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.8654708520179372,
-            "count": 223
+            "accuracy": 0.8454545454545455,
+            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.47,
-        "n_examples": 100,
+        "full_accuracy": 0.52,
+        "digit_accuracy": 0.8342857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 104
+            "count": 48
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 106
+            "count": 52
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 23
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.75625,
-            "count": 160
+            "accuracy": 0.7415730337078652,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.7035830618892508,
-            "count": 307
+            "accuracy": 0.7727272727272727,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.35,
-        "n_examples": 100,
+        "full_accuracy": 0.26,
+        "digit_accuracy": 0.6914285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.64,
-            "count": 100
+            "accuracy": 0.58,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.6075,
-            "count": 400
+            "accuracy": 0.565,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.51,
-        "n_examples": 100,
+        "full_accuracy": 0.44,
+        "digit_accuracy": 0.68,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.51,
-            "count": 100
+            "accuracy": 0.44,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.736,
-            "count": 500
+            "accuracy": 0.664,
+            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.86,
+        "full_accuracy": 0.88,
+        "digit_accuracy": 0.9792857142857143,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9821029082774049,
-            "count": 447
+            "accuracy": 0.9930394431554525,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.98125,
-            "count": 320
+            "accuracy": 0.9778481012658228,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.9642857142857143,
-            "count": 56
+            "accuracy": 0.9487179487179487,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.9735349716446124,
-            "count": 529
+            "accuracy": 0.975,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.8958333333333334,
-            "count": 48
+            "accuracy": 0.9444444444444444,
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.9,
+        "digit_accuracy": 0.9857142857142858,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 0.98,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.92,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.9,
+        "digit_accuracy": 0.98,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.995,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.9518072289156626,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 0.8823529411764706,
+            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.77,
-        "n_examples": 100,
+        "full_accuracy": 0.66,
+        "digit_accuracy": 0.9285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 300
+            "count": 150
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8911917098445595,
-            "count": 193
+            "accuracy": 0.86,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.897196261682243,
-            "count": 107
+            "accuracy": 0.78,
+            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.55,
-        "n_examples": 100,
+        "full_accuracy": 0.64,
+        "digit_accuracy": 0.94,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "SC": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.859375,
-            "count": 256
+            "accuracy": 0.8787878787878788,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.8333333333333334,
-            "count": 144
+            "accuracy": 0.9264705882352942,
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.57,
-        "n_examples": 100,
+        "full_accuracy": 0.64,
+        "digit_accuracy": 0.9142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8660130718954249,
-            "count": 306
+            "accuracy": 0.8835616438356164,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.7938144329896907,
-            "count": 194
+            "accuracy": 0.875,
+            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.8,
-        "n_examples": 100,
+        "full_accuracy": 0.78,
+        "digit_accuracy": 0.94,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.9562841530054644,
-            "count": 366
+            "accuracy": 0.9470899470899471,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.9273504273504274,
-            "count": 234
+            "accuracy": 0.9009009009009009,
+            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.96,
-        "n_examples": 100,
+        "full_accuracy": 0.92,
+        "digit_accuracy": 0.9885714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9933444259567388,
-            "count": 601
+            "accuracy": 0.9867986798679867,
+            "count": 303
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 99
+            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.9,
-        "n_examples": 100,
+        "full_accuracy": 0.94,
+        "digit_accuracy": 0.9885714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.992831541218638,
-            "count": 279
+            "accuracy": 1.0,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.9862068965517241,
-            "count": 145
+            "accuracy": 0.9722222222222222,
+            "count": 72
           },
           "ME": {
-            "accuracy": 0.9583333333333334,
-            "count": 24
+            "accuracy": 0.9444444444444444,
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.9761904761904762,
-            "count": 252
+            "accuracy": 0.9915966386554622,
+            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.67,
-        "n_examples": 100,
+        "full_accuracy": 0.62,
+        "digit_accuracy": 0.9371428571428572,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 213
+            "count": 112
           },
           "MB": {
-            "accuracy": 1.0,
-            "count": 113
+            "accuracy": 0.9622641509433962,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.9882352941176471,
-            "count": 85
+            "accuracy": 1.0,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.8232044198895028,
-            "count": 181
+            "accuracy": 0.8,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.9629629629629629,
-            "count": 108
+            "accuracy": 0.9433962264150944,
+            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.37,
-        "n_examples": 100,
+        "full_accuracy": 0.28,
+        "digit_accuracy": 0.8514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 179
+            "accuracy": 0.9896907216494846,
+            "count": 97
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 103
+            "count": 51
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.6442953020134228,
-            "count": 149
+            "accuracy": 0.5945945945945946,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.7511737089201878,
-            "count": 213
+            "accuracy": 0.7920792079207921,
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.26,
-        "n_examples": 100,
+        "full_accuracy": 0.28,
+        "digit_accuracy": 0.78,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.36,
-            "count": 100
+            "accuracy": 0.34,
+            "count": 50
           },
           "UD": {
             "accuracy": 0.7066666666666667,
-            "count": 300
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.23,
-        "n_examples": 100,
+        "full_accuracy": 0.28,
+        "digit_accuracy": 0.7771428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.33,
-            "count": 100
+            "accuracy": 0.36,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.695,
-            "count": 400
+            "accuracy": 0.77,
+            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.855,
+        "full_accuracy": 0.9,
+        "digit_accuracy": 0.985,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.99,
-            "count": 600
+            "accuracy": 0.9947368421052631,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.9812734082397003,
-            "count": 267
+            "accuracy": 0.9819494584837545,
+            "count": 277
           },
           "ME": {
-            "accuracy": 0.9811320754716981,
+            "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.958997722095672,
-            "count": 439
+            "accuracy": 0.9745222929936306,
+            "count": 471
           },
           "UD": {
-            "accuracy": 1.0,
-            "count": 41
+            "accuracy": 0.9655172413793104,
+            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.56,
-        "n_examples": 100,
+        "full_accuracy": 0.66,
+        "digit_accuracy": 0.9257142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9966666666666667,
-            "count": 300
+            "accuracy": 0.9933333333333333,
+            "count": 150
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.7868020304568528,
-            "count": 197
+            "accuracy": 0.8613861386138614,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.8349514563106796,
-            "count": 103
+            "accuracy": 0.7755102040816326,
+            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.51,
-        "n_examples": 100,
+        "full_accuracy": 0.46,
+        "digit_accuracy": 0.8885714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.8178137651821862,
-            "count": 247
+            "accuracy": 0.7768595041322314,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.7908496732026143,
-            "count": 153
+            "accuracy": 0.8481012658227848,
+            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.68,
-        "n_examples": 100,
+        "full_accuracy": 0.62,
+        "digit_accuracy": 0.9314285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.8959731543624161,
-            "count": 298
+            "accuracy": 0.8881578947368421,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.9257425742574258,
-            "count": 202
+            "accuracy": 0.9285714285714286,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.6520833333333333,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.6913333333333334,
+      "digit_accuracy": 0.9194285714285715,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_eval": {
@@ -2974,416 +3039,477 @@
       "K": 4,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 605
+            "count": 295
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 95
+            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 204
+            "count": 126
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 169
+            "count": 79
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 31
+            "count": 21
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 296
+            "count": 124
           }
         }
       },
       "add_S2": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 163
+            "count": 75
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 130
+            "count": 62
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 87
+            "count": 39
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 203
+            "count": 111
           },
           "US": {
             "accuracy": 1.0,
-            "count": 117
+            "count": 63
           }
         }
       },
       "add_S3": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 121
+            "count": 60
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 121
+            "count": 57
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 49
+            "count": 19
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 186
+            "count": 104
           },
           "US": {
             "accuracy": 1.0,
-            "count": 223
+            "count": 110
           }
         }
       },
       "add_S4": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 104
+            "count": 48
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 106
+            "count": 52
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 23
+            "count": 7
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 160
+            "count": 89
           },
           "US": {
             "accuracy": 1.0,
-            "count": 307
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.85,
-        "n_examples": 100,
+        "full_accuracy": 0.78,
+        "digit_accuracy": 0.9685714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.85,
-            "count": 100
+            "accuracy": 0.78,
+            "count": 50
           },
           "US": {
             "accuracy": 1.0,
-            "count": 400
+            "count": 200
           }
         }
       },
       "add_S6": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "US": {
             "accuracy": 1.0,
-            "count": 500
+            "count": 250
           }
         }
       },
       "add_random": {
         "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 447
+            "count": 431
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 320
+            "count": 316
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 39
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 529
+            "count": 560
           },
           "US": {
             "accuracy": 1.0,
-            "count": 48
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 17
           }
         }
       },
       "add_C3": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 300
+            "count": 150
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 193
+            "count": 100
           },
           "US": {
             "accuracy": 1.0,
-            "count": 107
+            "count": 50
           }
         }
       },
       "add_C4": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 256
+            "count": 132
           },
           "US": {
             "accuracy": 1.0,
-            "count": 144
+            "count": 68
           }
         }
       },
       "add_C5": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 306
+            "count": 146
           },
           "US": {
             "accuracy": 1.0,
-            "count": 194
+            "count": 104
           }
         }
       },
       "add_C6": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 366
+            "count": 189
           },
           "US": {
             "accuracy": 1.0,
-            "count": 234
+            "count": 111
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 601
+            "count": 303
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 99
+            "count": 47
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 279
+            "count": 141
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 145
+            "count": 72
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 24
+            "count": 18
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 252
+            "count": 119
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 213
+            "count": 112
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 113
+            "count": 53
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 85
+            "count": 47
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 181
+            "count": 85
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 108
+            "count": 53
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 179
+            "count": 97
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 103
+            "count": 51
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 27
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 149
+            "count": 74
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 213
+            "count": 101
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 300
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.99,
-        "n_examples": 100,
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 0.98,
+            "count": 50
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 400
+            "count": 200
           }
         }
       },
       "sub_random": {
         "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 600
+            "count": 570
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 267
+            "count": 277
           },
           "ME": {
             "accuracy": 1.0,
@@ -3391,85 +3517,89 @@
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 439
+            "count": 471
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 41
+            "count": 29
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 300
+            "count": 150
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 197
+            "count": 101
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 103
+            "count": 49
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 247
+            "count": 121
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 153
+            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.99,
-        "n_examples": 100,
+        "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.9966442953020134,
-            "count": 298
+            "accuracy": 1.0,
+            "count": 152
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 202
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.9929166666666667,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.992,
+      "digit_accuracy": 0.9987619047619047,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_overall_accuracy": 0.9929166666666667,