Upload add_sub_sorl_v1_abs30_10K/metrics.json with huggingface_hub

add_sub_sorl_v1_abs30_10K/metrics.json

@@ -290,502 +290,567 @@
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3628571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.2528925619834711,
-            "count": 605
+            "accuracy": 0.2711864406779661,
+            "count": 295
           },
           "SS": {
-            "accuracy": 0.8315789473684211,
-            "count": 95
+            "accuracy": 0.8545454545454545,
+            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.22857142857142856,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.30392156862745096,
-            "count": 204
+            "accuracy": 0.2857142857142857,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.21893491124260356,
-            "count": 169
+            "accuracy": 0.12658227848101267,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.7096774193548387,
-            "count": 31
+            "accuracy": 0.3333333333333333,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.27702702702702703,
-            "count": 296
+            "accuracy": 0.21774193548387097,
+            "count": 124
           }
         }
       },
       "add_S2": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3457142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.3312883435582822,
-            "count": 163
+            "accuracy": 0.32,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.13846153846153847,
-            "count": 130
+            "accuracy": 0.16129032258064516,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.4482758620689655,
-            "count": 87
+            "accuracy": 0.358974358974359,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.3842364532019704,
-            "count": 203
+            "accuracy": 0.35135135135135137,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.5299145299145299,
-            "count": 117
+            "accuracy": 0.5396825396825397,
+            "count": 63
           }
         }
       },
       "add_S3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.39714285714285713,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.36363636363636365,
-            "count": 121
+            "accuracy": 0.4166666666666667,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.06611570247933884,
-            "count": 121
+            "accuracy": 0.10526315789473684,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.42857142857142855,
-            "count": 49
+            "accuracy": 0.631578947368421,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.4032258064516129,
-            "count": 186
+            "accuracy": 0.3942307692307692,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.5919282511210763,
-            "count": 223
+            "accuracy": 0.5,
+            "count": 110
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.41714285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.38461538461538464,
-            "count": 104
+            "accuracy": 0.3541666666666667,
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.11320754716981132,
-            "count": 106
+            "accuracy": 0.11538461538461539,
+            "count": 52
           },
           "SS": {
-            "accuracy": 0.4782608695652174,
-            "count": 23
+            "accuracy": 0.5714285714285714,
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.41875,
-            "count": 160
+            "accuracy": 0.3595505617977528,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.49185667752442996,
-            "count": 307
+            "accuracy": 0.564935064935065,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.03,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.34,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.47,
-            "count": 100
+            "accuracy": 0.54,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.09,
-            "count": 100
+            "accuracy": 0.12,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.43,
-            "count": 100
+            "accuracy": 0.44,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.3375,
-            "count": 400
+            "accuracy": 0.32,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.12,
-        "n_examples": 100,
+        "full_accuracy": 0.18,
+        "digit_accuracy": 0.3514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.12,
-            "count": 100
+            "accuracy": 0.18,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.64,
-            "count": 100
+            "accuracy": 0.38,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.636,
-            "count": 500
+            "accuracy": 0.38,
+            "count": 250
           }
         }
       },
       "add_random": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.26785714285714285,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.3087248322147651,
-            "count": 447
+            "accuracy": 0.2691415313225058,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.165625,
-            "count": 320
+            "accuracy": 0.14556962025316456,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.5892857142857143,
-            "count": 56
+            "accuracy": 0.6410256410256411,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.29300567107750475,
-            "count": 529
+            "accuracy": 0.29464285714285715,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.4791666666666667,
-            "count": 48
+            "accuracy": 0.42592592592592593,
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.21428571428571427,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.256,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 0.12,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.1,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.24857142857142858,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.255,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.14,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.26506024096385544,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 0.4117647058823529,
+            "count": 17
           }
         }
       },
       "add_C3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.26285714285714284,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.35,
-            "count": 300
+            "accuracy": 0.30666666666666664,
+            "count": 150
           },
           "SC": {
             "accuracy": 0.1,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.21243523316062177,
-            "count": 193
+            "accuracy": 0.23,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.35514018691588783,
-            "count": 107
+            "accuracy": 0.36,
+            "count": 50
           }
         }
       },
       "add_C4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.28,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.46,
-            "count": 200
+            "accuracy": 0.47,
+            "count": 100
           },
           "SC": {
-            "accuracy": 0.07,
-            "count": 100
+            "accuracy": 0.08,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.15234375,
-            "count": 256
+            "accuracy": 0.13636363636363635,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.2916666666666667,
-            "count": 144
+            "accuracy": 0.4264705882352941,
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.02,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.26,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.57,
-            "count": 100
+            "accuracy": 0.4,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.15,
-            "count": 100
+            "accuracy": 0.04,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.2647058823529412,
-            "count": 306
+            "accuracy": 0.2328767123287671,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.4793814432989691,
-            "count": 194
+            "accuracy": 0.33653846153846156,
+            "count": 104
           }
         }
       },
       "add_C6": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.4085714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.16,
-            "count": 100
+            "accuracy": 0.14,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.28688524590163933,
-            "count": 366
+            "accuracy": 0.31216931216931215,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.6794871794871795,
-            "count": 234
+            "accuracy": 0.6936936936936937,
+            "count": 111
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3457142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.2579034941763727,
-            "count": 601
+            "accuracy": 0.26732673267326734,
+            "count": 303
           },
           "ME": {
-            "accuracy": 0.898989898989899,
-            "count": 99
+            "accuracy": 0.851063829787234,
+            "count": 47
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3171428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.44086021505376344,
-            "count": 279
+            "accuracy": 0.46099290780141844,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.013793103448275862,
-            "count": 145
+            "accuracy": 0.027777777777777776,
+            "count": 72
           },
           "ME": {
-            "accuracy": 0.875,
-            "count": 24
+            "accuracy": 1.0,
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.15873015873015872,
-            "count": 252
+            "accuracy": 0.2184873949579832,
+            "count": 119
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.4085714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.6525821596244131,
-            "count": 213
+            "accuracy": 0.6696428571428571,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.02654867256637168,
-            "count": 113
+            "accuracy": 0.018867924528301886,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.8941176470588236,
-            "count": 85
+            "accuracy": 0.9148936170212766,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.19889502762430938,
-            "count": 181
+            "accuracy": 0.21176470588235294,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.1111111111111111,
-            "count": 108
+            "accuracy": 0.11320754716981132,
+            "count": 53
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.32,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.7821229050279329,
-            "count": 179
+            "accuracy": 0.6804123711340206,
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.02912621359223301,
-            "count": 103
+            "accuracy": 0.0,
+            "count": 51
           },
           "ME": {
-            "accuracy": 0.8392857142857143,
-            "count": 56
+            "accuracy": 0.8518518518518519,
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.2550335570469799,
-            "count": 149
+            "accuracy": 0.20270270270270271,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.1267605633802817,
-            "count": 213
+            "accuracy": 0.07920792079207921,
+            "count": 101
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.29428571428571426,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.59,
-            "count": 200
+            "accuracy": 0.55,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.03,
-            "count": 100
+            "accuracy": 0.02,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.35,
-            "count": 100
+            "accuracy": 0.58,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.14333333333333334,
-            "count": 300
+            "accuracy": 0.12,
+            "count": 150
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.02,
-        "n_examples": 100,
+        "digit_accuracy": 0.29428571428571426,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.09,
-            "count": 100
+            "accuracy": 0.04,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.52,
-            "count": 100
+            "accuracy": 0.54,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.17,
-            "count": 400
+            "accuracy": 0.12,
+            "count": 200
           }
         }
       },
       "sub_random": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.27,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.4116666666666667,
-            "count": 600
+            "accuracy": 0.42105263157894735,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.02247191011235955,
-            "count": 267
+            "accuracy": 0.021660649819494584,
+            "count": 277
           },
           "ME": {
             "accuracy": 0.8679245283018868,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.1867881548974943,
-            "count": 439
+            "accuracy": 0.17834394904458598,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.1951219512195122,
-            "count": 41
+            "accuracy": 0.06896551724137931,
+            "count": 29
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.24,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.38333333333333336,
-            "count": 300
+            "accuracy": 0.38666666666666666,
+            "count": 150
           },
           "MB": {
-            "accuracy": 0.01,
-            "count": 100
+            "accuracy": 0.02,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.2131979695431472,
-            "count": 197
+            "accuracy": 0.2376237623762376,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.038834951456310676,
-            "count": 103
+            "accuracy": 0.02040816326530612,
+            "count": 49
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.24,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.555,
-            "count": 200
+            "accuracy": 0.6,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.02,
-            "count": 100
+            "accuracy": 0.04,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.21862348178137653,
-            "count": 247
+            "accuracy": 0.1652892561983471,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.0784313725490196,
-            "count": 153
+            "accuracy": 0.02531645569620253,
+            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.24571428571428572,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.03,
-            "count": 100
+            "accuracy": 0.02,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.174496644295302,
-            "count": 298
+            "accuracy": 0.17763157894736842,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.04455445544554455,
-            "count": 202
+            "accuracy": 0.08163265306122448,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.007916666666666667,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.006666666666666667,
+      "digit_accuracy": 0.29895238095238097,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_eval": {
@@ -794,502 +859,567 @@
       "K": 4,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.48,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.39669421487603307,
-            "count": 605
+            "accuracy": 0.39661016949152544,
+            "count": 295
           },
           "SS": {
-            "accuracy": 0.9263157894736842,
-            "count": 95
+            "accuracy": 0.9272727272727272,
+            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.35714285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.45098039215686275,
-            "count": 204
+            "accuracy": 0.48412698412698413,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.2603550295857988,
-            "count": 169
+            "accuracy": 0.13924050632911392,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.8709677419354839,
-            "count": 31
+            "accuracy": 0.6666666666666666,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.3141891891891892,
-            "count": 296
+            "accuracy": 0.31451612903225806,
+            "count": 124
           }
         }
       },
       "add_S2": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.4142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.4785276073619632,
-            "count": 163
+            "accuracy": 0.49333333333333335,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.12307692307692308,
-            "count": 130
+            "accuracy": 0.1774193548387097,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.3563218390804598,
-            "count": 87
+            "accuracy": 0.38461538461538464,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.4187192118226601,
-            "count": 203
+            "accuracy": 0.40540540540540543,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.5641025641025641,
-            "count": 117
+            "accuracy": 0.5873015873015873,
+            "count": 63
           }
         }
       },
       "add_S3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.42,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.512396694214876,
-            "count": 121
+            "accuracy": 0.5666666666666667,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.09917355371900827,
-            "count": 121
+            "accuracy": 0.08771929824561403,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.5102040816326531,
-            "count": 49
+            "accuracy": 0.7894736842105263,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.4032258064516129,
-            "count": 186
+            "accuracy": 0.40384615384615385,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.5560538116591929,
-            "count": 223
+            "accuracy": 0.4636363636363636,
+            "count": 110
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.44857142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.5865384615384616,
-            "count": 104
+            "accuracy": 0.4583333333333333,
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.11320754716981132,
-            "count": 106
+            "accuracy": 0.11538461538461539,
+            "count": 52
           },
           "SS": {
-            "accuracy": 0.6521739130434783,
-            "count": 23
+            "accuracy": 0.7142857142857143,
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.43125,
-            "count": 160
+            "accuracy": 0.42696629213483145,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.48534201954397393,
-            "count": 307
+            "accuracy": 0.5584415584415584,
+            "count": 154
           }
         }
       },
       "add_S5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.32,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.63,
-            "count": 100
+            "accuracy": 0.58,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.06,
-            "count": 100
+            "accuracy": 0.1,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.34,
-            "count": 100
+            "accuracy": 0.36,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.2575,
-            "count": 400
+            "accuracy": 0.3,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.1,
-        "n_examples": 100,
+        "full_accuracy": 0.18,
+        "digit_accuracy": 0.4657142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.1,
-            "count": 100
+            "accuracy": 0.18,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.53,
-            "count": 100
+            "accuracy": 0.54,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.506,
-            "count": 500
+            "accuracy": 0.508,
+            "count": 250
           }
         }
       },
       "add_random": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.35714285714285715,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.46308724832214765,
-            "count": 447
+            "accuracy": 0.4361948955916473,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.1875,
-            "count": 320
+            "accuracy": 0.17721518987341772,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.6785714285714286,
-            "count": 56
+            "accuracy": 0.7948717948717948,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.3856332703213611,
-            "count": 529
+            "accuracy": 0.35,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.4791666666666667,
-            "count": 48
+            "accuracy": 0.5370370370370371,
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.3342857142857143,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.416,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 0.12,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.14,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.3057142857142857,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.42,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.06,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.1927710843373494,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 0.23529411764705882,
+            "count": 17
           }
         }
       },
       "add_C3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.47333333333333333,
-            "count": 300
+            "accuracy": 0.52,
+            "count": 150
           },
           "SC": {
-            "accuracy": 0.04,
-            "count": 100
+            "accuracy": 0.1,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.21761658031088082,
-            "count": 193
+            "accuracy": 0.24,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.2336448598130841,
-            "count": 107
+            "accuracy": 0.32,
+            "count": 50
           }
         }
       },
       "add_C4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.35428571428571426,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.625,
-            "count": 200
+            "accuracy": 0.61,
+            "count": 100
           },
           "SC": {
-            "accuracy": 0.04,
-            "count": 100
+            "accuracy": 0.1,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.19140625,
-            "count": 256
+            "accuracy": 0.1893939393939394,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.2986111111111111,
-            "count": 144
+            "accuracy": 0.4852941176470588,
+            "count": 68
           }
         }
       },
       "add_C5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.36857142857142855,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.62,
-            "count": 100
+            "accuracy": 0.56,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.08,
-            "count": 100
+            "accuracy": 0.1,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.25163398692810457,
-            "count": 306
+            "accuracy": 0.2876712328767123,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.38144329896907214,
-            "count": 194
+            "accuracy": 0.5192307692307693,
+            "count": 104
           }
         }
       },
       "add_C6": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3628571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.14,
-            "count": 100
+            "accuracy": 0.06,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.32786885245901637,
-            "count": 366
+            "accuracy": 0.32275132275132273,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.7307692307692307,
-            "count": 234
+            "accuracy": 0.5675675675675675,
+            "count": 111
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.45714285714285713,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.36605657237936773,
-            "count": 601
+            "accuracy": 0.3795379537953795,
+            "count": 303
           },
           "ME": {
-            "accuracy": 0.8888888888888888,
-            "count": 99
+            "accuracy": 0.9574468085106383,
+            "count": 47
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.37142857142857144,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.5913978494623656,
-            "count": 279
+            "accuracy": 0.574468085106383,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.034482758620689655,
-            "count": 145
+            "accuracy": 0.027777777777777776,
+            "count": 72
           },
           "ME": {
-            "accuracy": 0.75,
-            "count": 24
+            "accuracy": 0.8333333333333334,
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.2222222222222222,
-            "count": 252
+            "accuracy": 0.2689075630252101,
+            "count": 119
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.4514285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.704225352112676,
-            "count": 213
+            "accuracy": 0.7232142857142857,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.04424778761061947,
-            "count": 113
+            "accuracy": 0.0,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.9647058823529412,
-            "count": 85
+            "accuracy": 0.9361702127659575,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.281767955801105,
-            "count": 181
+            "accuracy": 0.3058823529411765,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.12037037037037036,
-            "count": 108
+            "accuracy": 0.1320754716981132,
+            "count": 53
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3628571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.8156424581005587,
-            "count": 179
+            "accuracy": 0.7319587628865979,
+            "count": 97
           },
           "MB": {
             "accuracy": 0.0,
-            "count": 103
+            "count": 51
           },
           "ME": {
-            "accuracy": 0.9285714285714286,
-            "count": 56
+            "accuracy": 0.9629629629629629,
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.348993288590604,
-            "count": 149
+            "accuracy": 0.22972972972972974,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.0892018779342723,
-            "count": 213
+            "accuracy": 0.12871287128712872,
+            "count": 101
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.2914285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.76,
-            "count": 200
+            "accuracy": 0.66,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.01,
-            "count": 100
+            "accuracy": 0.0,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.44,
-            "count": 100
+            "accuracy": 0.72,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.08333333333333333,
-            "count": 300
+            "accuracy": 0.0,
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.01,
-        "n_examples": 100,
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.24285714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.04,
-            "count": 100
+            "accuracy": 0.0,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.7,
-            "count": 100
+            "accuracy": 0.58,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.065,
-            "count": 400
+            "accuracy": 0.03,
+            "count": 200
           }
         }
       },
       "sub_random": {
         "full_accuracy": 0.0,
+        "digit_accuracy": 0.35642857142857143,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.555,
-            "count": 600
+            "accuracy": 0.5508771929824562,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.03745318352059925,
-            "count": 267
+            "accuracy": 0.04693140794223827,
+            "count": 277
           },
           "ME": {
-            "accuracy": 0.9245283018867925,
+            "accuracy": 0.8113207547169812,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.275626423690205,
-            "count": 439
+            "accuracy": 0.25902335456475584,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.1951219512195122,
-            "count": 41
+            "accuracy": 0.2413793103448276,
+            "count": 29
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3314285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.55,
-            "count": 300
+            "accuracy": 0.5333333333333333,
+            "count": 150
           },
           "MB": {
-            "accuracy": 0.07,
-            "count": 100
+            "accuracy": 0.02,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.2233502538071066,
-            "count": 197
+            "accuracy": 0.27722772277227725,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.1941747572815534,
-            "count": 103
+            "accuracy": 0.14285714285714285,
+            "count": 49
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.68,
-            "count": 200
+            "accuracy": 0.74,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.03,
-            "count": 100
+            "accuracy": 0.0,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.2550607287449393,
-            "count": 247
+            "accuracy": 0.21487603305785125,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.1568627450980392,
-            "count": 153
+            "accuracy": 0.12658227848101267,
+            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "n_examples": 100,
+        "digit_accuracy": 0.3142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.06,
-            "count": 100
+            "accuracy": 0.04,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.31208053691275167,
-            "count": 298
+            "accuracy": 0.29605263157894735,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.19801980198019803,
-            "count": 202
+            "accuracy": 0.1326530612244898,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.004583333333333333,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.006,
+      "digit_accuracy": 0.36552380952380953,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_overall_accuracy": 0.004583333333333333,