Upload add_sub_sorl_v1_abs10_K1_25K_2L1H128d/metrics.json with huggingface_hub

add_sub_sorl_v1_abs10_K1_25K_2L1H128d/metrics.json

@@ -610,502 +610,567 @@
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3457142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.2066115702479339,
-            "count": 605
+            "accuracy": 0.22372881355932203,
+            "count": 295
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 95
+            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.19714285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.2549019607843137,
-            "count": 204
+            "accuracy": 0.25396825396825395,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.03550295857988166,
-            "count": 169
+            "accuracy": 0.05063291139240506,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.8064516129032258,
-            "count": 31
+            "accuracy": 0.8571428571428571,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.14527027027027026,
-            "count": 296
+            "accuracy": 0.12096774193548387,
+            "count": 124
           }
         }
       },
       "add_S2": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.37142857142857144,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.4662576687116564,
-            "count": 163
+            "accuracy": 0.48,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.07692307692307693,
-            "count": 130
+            "accuracy": 0.08064516129032258,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.5977011494252874,
-            "count": 87
+            "accuracy": 0.8717948717948718,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.26108374384236455,
-            "count": 203
+            "accuracy": 0.23423423423423423,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.4017094017094017,
-            "count": 117
+            "accuracy": 0.4603174603174603,
+            "count": 63
           }
         }
       },
       "add_S3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3057142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5206611570247934,
-            "count": 121
+            "accuracy": 0.5833333333333334,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.024793388429752067,
-            "count": 121
+            "accuracy": 0.05263157894736842,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.8571428571428571,
-            "count": 49
+            "accuracy": 0.8947368421052632,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.24731182795698925,
-            "count": 186
+            "accuracy": 0.16346153846153846,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.35874439461883406,
-            "count": 223
+            "accuracy": 0.3181818181818182,
+            "count": 110
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.29714285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5576923076923077,
-            "count": 104
+            "accuracy": 0.5208333333333334,
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.04716981132075472,
-            "count": 106
+            "accuracy": 0.057692307692307696,
+            "count": 52
           },
           "SS": {
-            "accuracy": 0.8695652173913043,
-            "count": 23
+            "accuracy": 0.7142857142857143,
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.25,
-            "count": 160
+            "accuracy": 0.21348314606741572,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.3583061889250814,
-            "count": 307
+            "accuracy": 0.33766233766233766,
+            "count": 154
           }
         }
       },
       "add_S5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.09142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.45,
-            "count": 100
+            "accuracy": 0.5,
+            "count": 50
           },
           "SC": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.11,
-            "count": 100
+            "accuracy": 0.06,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.04,
-            "count": 400
+            "accuracy": 0.02,
+            "count": 200
           }
         }
       },
       "add_S6": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.12,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.26,
-            "count": 100
+            "accuracy": 0.14,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.26,
-            "count": 500
+            "accuracy": 0.14,
+            "count": 250
           }
         }
       },
       "add_random": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.19857142857142857,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.27069351230425054,
-            "count": 447
+            "accuracy": 0.2482598607888631,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.053125,
-            "count": 320
+            "accuracy": 0.05379746835443038,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.7321428571428571,
-            "count": 56
+            "accuracy": 0.8461538461538461,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.166351606805293,
-            "count": 529
+            "accuracy": 0.17142857142857143,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.3333333333333333,
-            "count": 48
+            "accuracy": 0.46296296296296297,
+            "count": 54
           }
         }
       },
-      "add_C3": {
+      "add_C1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.12285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.24,
-            "count": 300
+            "accuracy": 0.172,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 0.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.0,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.12285714285714286,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.19,
+            "count": 200
           },
           "SC": {
             "accuracy": 0.02,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.06735751295336788,
-            "count": 193
+            "accuracy": 0.04819277108433735,
+            "count": 83
           },
           "US": {
-            "accuracy": 0.06542056074766354,
-            "count": 107
+            "accuracy": 0.0,
+            "count": 17
           }
         }
       },
-      "add_C4": {
+      "add_C3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.13428571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.335,
-            "count": 200
+            "accuracy": 0.24666666666666667,
+            "count": 150
           },
           "SC": {
-            "accuracy": 0.01,
+            "accuracy": 0.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.06,
             "count": 100
           },
+          "US": {
+            "accuracy": 0.08,
+            "count": 50
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.12857142857142856,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.35,
+            "count": 100
+          },
+          "SC": {
+            "accuracy": 0.0,
+            "count": 50
+          },
           "UC": {
-            "accuracy": 0.04296875,
-            "count": 256
+            "accuracy": 0.03787878787878788,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.06944444444444445,
-            "count": 144
+            "accuracy": 0.07352941176470588,
+            "count": 68
           }
         }
       },
       "add_C5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.21428571428571427,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.49,
-            "count": 100
+            "accuracy": 0.36,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.05,
-            "count": 100
+            "accuracy": 0.04,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.08169934640522876,
-            "count": 306
+            "accuracy": 0.13013698630136986,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.23195876288659795,
-            "count": 194
+            "accuracy": 0.34615384615384615,
+            "count": 104
           }
         }
       },
       "add_C6": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.01,
-            "count": 100
+            "accuracy": 0.0,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.1912568306010929,
-            "count": 366
+            "accuracy": 0.20105820105820105,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.6239316239316239,
-            "count": 234
+            "accuracy": 0.6036036036036037,
+            "count": 111
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3057142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.20465890183028287,
-            "count": 601
+            "accuracy": 0.19801980198019803,
+            "count": 303
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 99
+            "count": 47
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.26285714285714284,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.3835125448028674,
-            "count": 279
+            "accuracy": 0.3900709219858156,
+            "count": 141
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 145
+            "count": 72
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 24
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.09523809523809523,
-            "count": 252
+            "accuracy": 0.15966386554621848,
+            "count": 119
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.38285714285714284,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6150234741784038,
-            "count": 213
+            "accuracy": 0.6428571428571429,
+            "count": 112
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 113
+            "count": 53
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 85
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.16574585635359115,
-            "count": 181
+            "accuracy": 0.17647058823529413,
+            "count": 85
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 108
+            "count": 53
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.28,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.7597765363128491,
-            "count": 179
+            "accuracy": 0.6494845360824743,
+            "count": 97
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 103
+            "count": 51
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.12080536912751678,
-            "count": 149
+            "accuracy": 0.10810810810810811,
+            "count": 74
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 213
+            "count": 101
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.21142857142857144,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 0.5,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.3,
-            "count": 100
+            "accuracy": 0.48,
+            "count": 50
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 300
+            "count": 150
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.18285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.31,
-            "count": 100
+            "accuracy": 0.28,
+            "count": 50
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 400
+            "count": 200
           }
         }
       },
       "sub_random": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.2307142857142857,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.3616666666666667,
-            "count": 600
+            "accuracy": 0.37719298245614036,
+            "count": 570
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 267
+            "count": 277
           },
           "ME": {
             "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.12072892938496584,
-            "count": 439
+            "accuracy": 0.11677282377919321,
+            "count": 471
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 41
+            "count": 29
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.18857142857142858,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 0.3333333333333333,
-            "count": 300
+            "count": 150
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.17766497461928935,
-            "count": 197
+            "accuracy": 0.15841584158415842,
+            "count": 101
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 103
+            "count": 49
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.17714285714285713,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 0.5,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.145748987854251,
-            "count": 247
+            "accuracy": 0.09917355371900827,
+            "count": 121
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 153
+            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.18,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.11073825503355705,
-            "count": 298
+            "accuracy": 0.08552631578947369,
+            "count": 152
           },
           "UD": {
             "accuracy": 0.0,
-            "count": 202
+            "count": 98
           }
         }
       }
     },
     "summary": {
       "overall_accuracy": 0.0,
-      "total_examples": 2400,
-      "n_splits": 22
+      "digit_accuracy": 0.22114285714285714,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_eval": {
@@ -1114,502 +1179,567 @@
       "K": 1,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.09,
-        "n_examples": 100,
+        "full_accuracy": 0.08,
+        "digit_accuracy": 0.7114285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6776859504132231,
-            "count": 605
+            "accuracy": 0.688135593220339,
+            "count": 295
           },
           "SS": {
-            "accuracy": 0.8947368421052632,
-            "count": 95
+            "accuracy": 0.8363636363636363,
+            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.03,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.6228571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.696078431372549,
-            "count": 204
+            "accuracy": 0.7142857142857143,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.6804733727810651,
-            "count": 169
+            "accuracy": 0.7215189873417721,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.8387096774193549,
-            "count": 31
+            "accuracy": 0.8571428571428571,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.4831081081081081,
-            "count": 296
+            "accuracy": 0.4274193548387097,
+            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.05,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.5857142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.7423312883435583,
-            "count": 163
+            "accuracy": 0.7333333333333333,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.6615384615384615,
-            "count": 130
+            "accuracy": 0.7096774193548387,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.7471264367816092,
-            "count": 87
+            "accuracy": 0.7435897435897436,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.37438423645320196,
-            "count": 203
+            "accuracy": 0.36936936936936937,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.49572649572649574,
-            "count": 117
+            "accuracy": 0.5714285714285714,
+            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.5485714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.743801652892562,
-            "count": 121
+            "accuracy": 0.7833333333333333,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.6611570247933884,
-            "count": 121
+            "accuracy": 0.631578947368421,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.8367346938775511,
-            "count": 49
+            "accuracy": 0.7894736842105263,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.3709677419354839,
-            "count": 186
+            "accuracy": 0.3942307692307692,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.3811659192825112,
-            "count": 223
+            "accuracy": 0.4818181818181818,
+            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.03,
-        "n_examples": 100,
+        "full_accuracy": 0.04,
+        "digit_accuracy": 0.46,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.7307692307692307,
-            "count": 104
+            "accuracy": 0.7708333333333334,
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.7075471698113207,
-            "count": 106
+            "accuracy": 0.6923076923076923,
+            "count": 52
           },
           "SS": {
-            "accuracy": 0.9565217391304348,
-            "count": 23
+            "accuracy": 0.7142857142857143,
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.325,
-            "count": 160
+            "accuracy": 0.3258426966292135,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.3322475570032573,
-            "count": 307
+            "accuracy": 0.35064935064935066,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.03,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.3314285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.65,
-            "count": 100
+            "accuracy": 0.68,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.63,
-            "count": 100
+            "accuracy": 0.72,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.13,
-            "count": 100
+            "accuracy": 0.12,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.285,
-            "count": 400
+            "accuracy": 0.2,
+            "count": 200
           }
         }
       },
       "add_S6": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.09142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.48,
-            "count": 100
+            "accuracy": 0.52,
+            "count": 50
           },
           "UC": {
             "accuracy": 0.0,
-            "count": 100
+            "count": 50
           },
           "US": {
-            "accuracy": 0.036,
-            "count": 500
+            "accuracy": 0.024,
+            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.04,
+        "full_accuracy": 0.025,
+        "digit_accuracy": 0.5957142857142858,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.6487695749440716,
-            "count": 447
+            "accuracy": 0.6682134570765661,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.70625,
-            "count": 320
+            "accuracy": 0.6930379746835443,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.8571428571428571,
-            "count": 56
+            "accuracy": 0.8974358974358975,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.46313799621928164,
-            "count": 529
+            "accuracy": 0.4589285714285714,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.625,
-            "count": 48
+            "accuracy": 0.6481481481481481,
+            "count": 54
           }
         }
       },
-      "add_C3": {
+      "add_C1": {
         "full_accuracy": 0.02,
-        "n_examples": 100,
+        "digit_accuracy": 0.7228571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.8033333333333333,
-            "count": 300
+            "accuracy": 0.82,
+            "count": 250
           },
           "SC": {
             "accuracy": 0.78,
-            "count": 100
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.18,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.6485714285714286,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.81,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.76,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.19170984455958548,
-            "count": 193
+            "accuracy": 0.20481927710843373,
+            "count": 83
           },
           "US": {
-            "accuracy": 0.5327102803738317,
-            "count": 107
+            "accuracy": 0.5882352941176471,
+            "count": 17
           }
         }
       },
-      "add_C4": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+      "add_C3": {
+        "full_accuracy": 0.04,
+        "digit_accuracy": 0.62,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 0.8,
-            "count": 200
+            "count": 150
           },
           "SC": {
-            "accuracy": 0.75,
+            "accuracy": 0.78,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.28,
+            "count": 100
+          },
+          "US": {
+            "accuracy": 0.6,
+            "count": 50
+          }
+        }
+      },
+      "add_C4": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.52,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.76,
             "count": 100
           },
+          "SC": {
+            "accuracy": 0.68,
+            "count": 50
+          },
           "UC": {
-            "accuracy": 0.1875,
-            "count": 256
+            "accuracy": 0.15151515151515152,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.5833333333333334,
-            "count": 144
+            "accuracy": 0.7647058823529411,
+            "count": 68
           }
         }
       },
       "add_C5": {
         "full_accuracy": 0.02,
-        "n_examples": 100,
+        "digit_accuracy": 0.5,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.79,
-            "count": 100
+            "accuracy": 0.76,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.65,
-            "count": 100
+            "accuracy": 0.72,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.23529411764705882,
-            "count": 306
+            "accuracy": 0.2465753424657534,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.5979381443298969,
-            "count": 194
+            "accuracy": 0.625,
+            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.44571428571428573,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.6,
-            "count": 100
+            "accuracy": 0.66,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.319672131147541,
-            "count": 366
+            "accuracy": 0.37037037037037035,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.6282051282051282,
-            "count": 234
+            "accuracy": 0.4774774774774775,
+            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.06,
+        "digit_accuracy": 0.6371428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.5391014975041597,
-            "count": 601
+            "accuracy": 0.5808580858085809,
+            "count": 303
           },
           "ME": {
-            "accuracy": 0.9595959595959596,
-            "count": 99
+            "accuracy": 1.0,
+            "count": 47
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.5828571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6666666666666666,
-            "count": 279
+            "accuracy": 0.6453900709219859,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.38620689655172413,
-            "count": 145
+            "accuracy": 0.4027777777777778,
+            "count": 72
           },
           "ME": {
-            "accuracy": 0.9583333333333334,
-            "count": 24
+            "accuracy": 0.8888888888888888,
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.5198412698412699,
-            "count": 252
+            "accuracy": 0.5714285714285714,
+            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.0,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.5885714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.755868544600939,
-            "count": 213
+            "accuracy": 0.8035714285714286,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.26548672566371684,
-            "count": 113
+            "accuracy": 0.33962264150943394,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.9764705882352941,
-            "count": 85
+            "accuracy": 0.9574468085106383,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.5524861878453039,
-            "count": 181
+            "accuracy": 0.5058823529411764,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.2037037037037037,
-            "count": 108
+            "accuracy": 0.18867924528301888,
+            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.44857142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8547486033519553,
-            "count": 179
+            "accuracy": 0.7628865979381443,
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.3300970873786408,
-            "count": 103
+            "accuracy": 0.19607843137254902,
+            "count": 51
           },
           "ME": {
-            "accuracy": 0.9821428571428571,
-            "count": 56
+            "accuracy": 0.9629629629629629,
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.5838926174496645,
-            "count": 149
+            "accuracy": 0.5675675675675675,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.08450704225352113,
-            "count": 213
+            "accuracy": 0.04950495049504951,
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.3485714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.65,
-            "count": 200
+            "accuracy": 0.62,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.21,
-            "count": 100
+            "accuracy": 0.18,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.67,
-            "count": 100
+            "accuracy": 0.78,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.06333333333333334,
-            "count": 300
+            "accuracy": 0.08,
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.06,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.28285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.13,
-            "count": 100
+            "accuracy": 0.02,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.72,
-            "count": 100
+            "accuracy": 0.74,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.145,
-            "count": 400
+            "accuracy": 0.055,
+            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.01,
+        "full_accuracy": 0.025,
+        "digit_accuracy": 0.5657142857142857,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.635,
-            "count": 600
+            "accuracy": 0.6385964912280702,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.45318352059925093,
-            "count": 267
+            "accuracy": 0.45126353790613716,
+            "count": 277
           },
           "ME": {
-            "accuracy": 0.9811320754716981,
+            "accuracy": 0.9245283018867925,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.5444191343963554,
-            "count": 439
+            "accuracy": 0.5329087048832272,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.1951219512195122,
-            "count": 41
+            "accuracy": 0.10344827586206896,
+            "count": 29
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.4542857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.5266666666666666,
-            "count": 300
+            "accuracy": 0.5666666666666667,
+            "count": 150
           },
           "MB": {
-            "accuracy": 0.41,
-            "count": 100
+            "accuracy": 0.36,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.49746192893401014,
-            "count": 197
+            "accuracy": 0.5148514851485149,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.11650485436893204,
-            "count": 103
+            "accuracy": 0.08163265306122448,
+            "count": 49
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.4342857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.67,
-            "count": 200
+            "accuracy": 0.6,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.35,
-            "count": 100
+            "accuracy": 0.36,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.4898785425101215,
-            "count": 247
+            "accuracy": 0.4793388429752066,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.16993464052287582,
-            "count": 153
+            "accuracy": 0.20253164556962025,
+            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.46,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.15,
-            "count": 100
+            "accuracy": 0.2,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.4395973154362416,
-            "count": 298
+            "accuracy": 0.4407894736842105,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.1782178217821782,
-            "count": 202
+            "accuracy": 0.3469387755102041,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.020416666666666666,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.019333333333333334,
+      "digit_accuracy": 0.522952380952381,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_overall_accuracy": 0.020416666666666666,