Upload add_sub_sorl_v1_abs10_25K/metrics.json with huggingface_hub

add_sub_sorl_v1_abs10_25K/metrics.json

@@ -610,502 +610,567 @@
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.55,
-        "n_examples": 100,
+        "full_accuracy": 0.48,
+        "digit_accuracy": 0.9085714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9190082644628099,
-            "count": 605
+            "accuracy": 0.9050847457627119,
+            "count": 295
           },
           "SS": {
-            "accuracy": 0.9473684210526315,
-            "count": 95
+            "accuracy": 0.9272727272727272,
+            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 0.62,
-        "n_examples": 100,
+        "digit_accuracy": 0.9314285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9215686274509803,
-            "count": 204
+            "accuracy": 0.9365079365079365,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.9467455621301775,
-            "count": 169
+            "accuracy": 0.9493670886075949,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.9032258064516129,
-            "count": 31
+            "accuracy": 0.9523809523809523,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.9391891891891891,
-            "count": 296
+            "accuracy": 0.9112903225806451,
+            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.54,
-        "n_examples": 100,
+        "full_accuracy": 0.28,
+        "digit_accuracy": 0.8371428571428572,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9447852760736196,
-            "count": 163
+            "accuracy": 0.92,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.8769230769230769,
-            "count": 130
+            "accuracy": 0.8387096774193549,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.9310344827586207,
-            "count": 87
+            "accuracy": 0.6410256410256411,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.8472906403940886,
-            "count": 203
+            "accuracy": 0.8198198198198198,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.9572649572649573,
-            "count": 117
+            "accuracy": 0.8888888888888888,
+            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.42,
-        "n_examples": 100,
+        "full_accuracy": 0.46,
+        "digit_accuracy": 0.8571428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9504132231404959,
-            "count": 121
+            "accuracy": 0.9333333333333333,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.9338842975206612,
-            "count": 121
+            "accuracy": 0.8245614035087719,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.8775510204081632,
-            "count": 49
+            "accuracy": 0.8421052631578947,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.7956989247311828,
-            "count": 186
+            "accuracy": 0.8461538461538461,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.8609865470852018,
-            "count": 223
+            "accuracy": 0.8454545454545455,
+            "count": 110
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.32,
-        "n_examples": 100,
+        "digit_accuracy": 0.7428571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9519230769230769,
-            "count": 104
+            "accuracy": 1.0,
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.8962264150943396,
-            "count": 106
+            "accuracy": 0.8846153846153846,
+            "count": 52
           },
           "SS": {
-            "accuracy": 0.8695652173913043,
-            "count": 23
+            "accuracy": 0.7142857142857143,
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.70625,
-            "count": 160
+            "accuracy": 0.6629213483146067,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.6710097719869706,
-            "count": 307
+            "accuracy": 0.6623376623376623,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.26,
-        "n_examples": 100,
+        "full_accuracy": 0.34,
+        "digit_accuracy": 0.6285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.93,
-            "count": 100
+            "accuracy": 0.94,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.47,
-            "count": 100
+            "accuracy": 0.5,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.44,
-            "count": 400
+            "accuracy": 0.49,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.41,
-        "n_examples": 100,
+        "full_accuracy": 0.36,
+        "digit_accuracy": 0.5114285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.43,
-            "count": 100
+            "accuracy": 0.38,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.516,
-            "count": 500
+            "accuracy": 0.44,
+            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.7,
+        "full_accuracy": 0.715,
+        "digit_accuracy": 0.9485714285714286,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.941834451901566,
-            "count": 447
+            "accuracy": 0.9559164733178654,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.94375,
-            "count": 320
+            "accuracy": 0.9651898734177216,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.9642857142857143,
-            "count": 56
+            "accuracy": 0.9230769230769231,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.9640831758034026,
-            "count": 529
+            "accuracy": 0.9357142857142857,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.8125,
-            "count": 48
+            "accuracy": 0.9444444444444444,
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.66,
+        "digit_accuracy": 0.94,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.992,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 0.9,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.72,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.78,
+        "digit_accuracy": 0.96,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.995,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.98,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.9156626506024096,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 0.7058823529411765,
+            "count": 17
           }
         }
       },
       "add_C3": {
         "full_accuracy": 0.52,
-        "n_examples": 100,
+        "digit_accuracy": 0.8828571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.99,
-            "count": 300
+            "accuracy": 0.9866666666666667,
+            "count": 150
           },
           "SC": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 0.86,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.7927461139896373,
-            "count": 193
+            "accuracy": 0.84,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.7570093457943925,
-            "count": 107
+            "accuracy": 0.68,
+            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.53,
-        "n_examples": 100,
+        "full_accuracy": 0.56,
+        "digit_accuracy": 0.9057142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 0.99,
-            "count": 200
+            "count": 100
           },
           "SC": {
-            "accuracy": 0.95,
-            "count": 100
+            "accuracy": 1.0,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8515625,
-            "count": 256
+            "accuracy": 0.8257575757575758,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.7916666666666666,
-            "count": 144
+            "accuracy": 0.8676470588235294,
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.47,
-        "n_examples": 100,
+        "full_accuracy": 0.46,
+        "digit_accuracy": 0.8971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.97,
-            "count": 100
+            "accuracy": 0.96,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8235294117647058,
-            "count": 306
+            "accuracy": 0.8356164383561644,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.845360824742268,
-            "count": 194
+            "accuracy": 0.9038461538461539,
+            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.41,
-        "n_examples": 100,
+        "full_accuracy": 0.48,
+        "digit_accuracy": 0.8714285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.825136612021858,
-            "count": 366
+            "accuracy": 0.8518518518518519,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.8803418803418803,
-            "count": 234
+            "accuracy": 0.8468468468468469,
+            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.8,
-        "n_examples": 100,
+        "full_accuracy": 0.92,
+        "digit_accuracy": 0.9885714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9683860232945092,
-            "count": 601
+            "accuracy": 0.9867986798679867,
+            "count": 303
           },
           "ME": {
-            "accuracy": 0.9797979797979798,
-            "count": 99
+            "accuracy": 1.0,
+            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.67,
-        "n_examples": 100,
+        "full_accuracy": 0.6,
+        "digit_accuracy": 0.9314285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.982078853046595,
-            "count": 279
+            "accuracy": 0.9645390070921985,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.9379310344827586,
-            "count": 145
+            "accuracy": 0.9027777777777778,
+            "count": 72
           },
           "ME": {
-            "accuracy": 0.9583333333333334,
-            "count": 24
+            "accuracy": 1.0,
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.9047619047619048,
-            "count": 252
+            "accuracy": 0.8991596638655462,
+            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.38,
-        "n_examples": 100,
+        "full_accuracy": 0.32,
+        "digit_accuracy": 0.86,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9906103286384976,
-            "count": 213
+            "accuracy": 0.9732142857142857,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.9380530973451328,
-            "count": 113
+            "accuracy": 0.9056603773584906,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.9882352941176471,
-            "count": 85
+            "accuracy": 1.0,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.6961325966850829,
-            "count": 181
+            "accuracy": 0.611764705882353,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.8981481481481481,
-            "count": 108
+            "accuracy": 0.8490566037735849,
+            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.12,
-        "n_examples": 100,
+        "full_accuracy": 0.1,
+        "digit_accuracy": 0.7514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 179
+            "accuracy": 0.9896907216494846,
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.9223300970873787,
-            "count": 103
+            "accuracy": 0.9803921568627451,
+            "count": 51
           },
           "ME": {
-            "accuracy": 0.9821428571428571,
-            "count": 56
+            "accuracy": 1.0,
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.48322147651006714,
-            "count": 149
+            "accuracy": 0.47297297297297297,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.6291079812206573,
-            "count": 213
+            "accuracy": 0.5445544554455446,
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.06,
-        "n_examples": 100,
+        "full_accuracy": 0.04,
+        "digit_accuracy": 0.6457142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 0.98,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.3,
-            "count": 100
+            "accuracy": 0.28,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.38666666666666666,
-            "count": 300
+            "accuracy": 0.42,
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.03,
-        "n_examples": 100,
+        "full_accuracy": 0.06,
+        "digit_accuracy": 0.48,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.19,
-            "count": 100
+            "accuracy": 0.26,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.2775,
-            "count": 400
+            "accuracy": 0.275,
+            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.655,
+        "full_accuracy": 0.635,
+        "digit_accuracy": 0.9385714285714286,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9833333333333333,
-            "count": 600
+            "accuracy": 0.9789473684210527,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.9625468164794008,
-            "count": 267
+            "accuracy": 0.9566787003610109,
+            "count": 277
           },
           "ME": {
-            "accuracy": 1.0,
+            "accuracy": 0.9811320754716981,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.876993166287016,
-            "count": 439
+            "accuracy": 0.8747346072186837,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.8780487804878049,
-            "count": 41
+            "accuracy": 0.9310344827586207,
+            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.35,
-        "n_examples": 100,
+        "full_accuracy": 0.4,
+        "digit_accuracy": 0.8685714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.99,
-            "count": 300
+            "accuracy": 0.9866666666666667,
+            "count": 150
           },
           "MB": {
-            "accuracy": 0.95,
-            "count": 100
+            "accuracy": 0.98,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.6751269035532995,
-            "count": 197
+            "accuracy": 0.7128712871287128,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.6893203883495146,
-            "count": 103
+            "accuracy": 0.7142857142857143,
+            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.24,
-        "n_examples": 100,
+        "full_accuracy": 0.28,
+        "digit_accuracy": 0.8228571428571428,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.995,
-            "count": 200
+            "accuracy": 1.0,
+            "count": 100
           },
           "MB": {
             "accuracy": 0.94,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.7206477732793523,
-            "count": 247
+            "accuracy": 0.7520661157024794,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.6209150326797386,
-            "count": 153
+            "accuracy": 0.6329113924050633,
+            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.21,
-        "n_examples": 100,
+        "full_accuracy": 0.22,
+        "digit_accuracy": 0.76,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.7013422818791947,
-            "count": 298
+            "accuracy": 0.7302631578947368,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.6782178217821783,
-            "count": 202
+            "accuracy": 0.5612244897959183,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.4425,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.4886666666666667,
+      "digit_accuracy": 0.851047619047619,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_eval": {
@@ -1114,416 +1179,477 @@
       "K": 4,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 605
+            "count": 295
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 95
+            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 204
+            "count": 126
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 169
+            "count": 79
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 31
+            "count": 21
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 296
+            "count": 124
           }
         }
       },
       "add_S2": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 163
+            "count": 75
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 130
+            "count": 62
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 87
+            "count": 39
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 203
+            "count": 111
           },
           "US": {
             "accuracy": 1.0,
-            "count": 117
+            "count": 63
           }
         }
       },
       "add_S3": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 121
+            "count": 60
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 121
+            "count": 57
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 49
+            "count": 19
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 186
+            "count": 104
           },
           "US": {
             "accuracy": 1.0,
-            "count": 223
+            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.95,
-        "n_examples": 100,
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 104
+            "count": 48
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 106
+            "count": 52
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 23
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.96875,
-            "count": 160
+            "accuracy": 0.9887640449438202,
+            "count": 89
           },
           "US": {
             "accuracy": 1.0,
-            "count": 307
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.79,
-        "n_examples": 100,
+        "full_accuracy": 0.74,
+        "digit_accuracy": 0.96,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8,
-            "count": 100
+            "accuracy": 0.76,
+            "count": 50
           },
           "US": {
             "accuracy": 0.99,
-            "count": 400
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.97,
-        "n_examples": 100,
+        "full_accuracy": 0.94,
+        "digit_accuracy": 0.9857142857142858,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.97,
-            "count": 100
+            "accuracy": 0.96,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.992,
-            "count": 500
+            "accuracy": 0.988,
+            "count": 250
           }
         }
       },
       "add_random": {
         "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 447
+            "count": 431
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 320
+            "count": 316
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 39
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 529
+            "count": 560
           },
           "US": {
             "accuracy": 1.0,
-            "count": 48
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 1.0,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 1.0,
+            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.99,
-        "n_examples": 100,
+        "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 300
+            "count": 150
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.9948186528497409,
-            "count": 193
+            "accuracy": 1.0,
+            "count": 100
           },
           "US": {
             "accuracy": 1.0,
-            "count": 107
+            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.99,
-        "n_examples": 100,
+        "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.99609375,
-            "count": 256
+            "accuracy": 1.0,
+            "count": 132
           },
           "US": {
             "accuracy": 1.0,
-            "count": 144
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.98,
-        "n_examples": 100,
+        "full_accuracy": 0.96,
+        "digit_accuracy": 0.9942857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.9934640522875817,
-            "count": 306
+            "accuracy": 0.9863013698630136,
+            "count": 146
           },
           "US": {
             "accuracy": 1.0,
-            "count": 194
+            "count": 104
           }
         }
       },
       "add_C6": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
             "accuracy": 1.0,
-            "count": 366
+            "count": 189
           },
           "US": {
             "accuracy": 1.0,
-            "count": 234
+            "count": 111
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 601
+            "count": 303
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 99
+            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 1.0,
-        "n_examples": 100,
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 279
+            "count": 141
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 145
+            "count": 72
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 24
+            "count": 18
           },
           "UB": {
-            "accuracy": 1.0,
-            "count": 252
+            "accuracy": 0.9915966386554622,
+            "count": 119
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 213
+            "count": 112
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 113
+            "count": 53
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 85
+            "count": 47
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 181
+            "count": 85
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 108
+            "count": 53
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 1.0,
-        "n_examples": 100,
+        "digit_accuracy": 1.0,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 179
+            "count": 97
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 103
+            "count": 51
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 27
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 149
+            "count": 74
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 213
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.85,
-        "n_examples": 100,
+        "full_accuracy": 0.82,
+        "digit_accuracy": 0.9742857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.85,
-            "count": 100
+            "accuracy": 0.82,
+            "count": 50
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 300
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.79,
-        "n_examples": 100,
+        "full_accuracy": 0.72,
+        "digit_accuracy": 0.96,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.82,
-            "count": 100
+            "accuracy": 0.78,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.99,
-            "count": 400
+            "accuracy": 0.985,
+            "count": 200
           }
         }
       },
       "sub_random": {
         "full_accuracy": 1.0,
+        "digit_accuracy": 1.0,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 600
+            "count": 570
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 267
+            "count": 277
           },
           "ME": {
             "accuracy": 1.0,
@@ -1531,85 +1657,89 @@
           },
           "UB": {
             "accuracy": 1.0,
-            "count": 439
+            "count": 471
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 41
+            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 1.0,
-        "n_examples": 100,
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 300
+            "count": 150
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 1.0,
-            "count": 197
+            "accuracy": 0.9900990099009901,
+            "count": 101
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 103
+            "count": 49
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.98,
-        "n_examples": 100,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.9919028340080972,
-            "count": 247
+            "accuracy": 0.9917355371900827,
+            "count": 121
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 153
+            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 1.0,
-        "n_examples": 100,
+        "full_accuracy": 0.98,
+        "digit_accuracy": 0.9971428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 1.0,
-            "count": 298
+            "accuracy": 0.993421052631579,
+            "count": 152
           },
           "UD": {
             "accuracy": 1.0,
-            "count": 202
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.9704166666666667,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.9693333333333334,
+      "digit_accuracy": 0.9953333333333333,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_overall_accuracy": 0.9704166666666667,