Upload add_sub_sorl_v6_abs30_K1_25K_1L3H510d/metrics.json with huggingface_hub

add_sub_sorl_v6_abs30_K1_25K_1L3H510d/metrics.json

@@ -1712,525 +1712,567 @@
       "K": null,
       "mode": "sft",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.08714285714285715,
-        "n_examples": 100,
+        "digit_accuracy": 0.08,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.09256198347107437,
-            "count": 605
+            "accuracy": 0.0847457627118644,
+            "count": 295
           },
           "SS": {
-            "accuracy": 0.05263157894736842,
-            "count": 95
+            "accuracy": 0.05454545454545454,
+            "count": 55
           }
         }
       },
       "add_S1": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.08571428571428572,
-        "n_examples": 100,
+        "digit_accuracy": 0.06571428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.07352941176470588,
-            "count": 204
+            "accuracy": 0.06349206349206349,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.07692307692307693,
-            "count": 169
+            "accuracy": 0.06329113924050633,
+            "count": 79
           },
           "SS": {
-            "accuracy": 0.03225806451612903,
-            "count": 31
+            "accuracy": 0.0,
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.10472972972972973,
-            "count": 296
+            "accuracy": 0.08064516129032258,
+            "count": 124
           }
         }
       },
       "add_S2": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.09571428571428571,
-        "n_examples": 100,
+        "digit_accuracy": 0.11142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.10429447852760736,
-            "count": 163
+            "accuracy": 0.12,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.1076923076923077,
-            "count": 130
+            "accuracy": 0.16129032258064516,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.06896551724137931,
-            "count": 87
+            "accuracy": 0.07692307692307693,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.09852216748768473,
-            "count": 203
+            "accuracy": 0.0990990990990991,
+            "count": 111
           },
           "US": {
-            "accuracy": 0.08547008547008547,
-            "count": 117
+            "accuracy": 0.09523809523809523,
+            "count": 63
           }
         }
       },
       "add_S3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.09428571428571429,
-        "n_examples": 100,
+        "digit_accuracy": 0.08857142857142856,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.10743801652892562,
-            "count": 121
+            "accuracy": 0.11666666666666667,
+            "count": 60
           },
           "SC": {
-            "accuracy": 0.11570247933884298,
-            "count": 121
+            "accuracy": 0.10526315789473684,
+            "count": 57
           },
           "SS": {
-            "accuracy": 0.061224489795918366,
-            "count": 49
+            "accuracy": 0.05263157894736842,
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.11290322580645161,
-            "count": 186
+            "accuracy": 0.10576923076923077,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.06726457399103139,
-            "count": 223
+            "accuracy": 0.05454545454545454,
+            "count": 110
           }
         }
       },
       "add_S4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.07428571428571429,
-        "n_examples": 100,
+        "digit_accuracy": 0.08285714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.0673076923076923,
-            "count": 104
+            "accuracy": 0.125,
+            "count": 48
           },
           "SC": {
-            "accuracy": 0.11320754716981132,
-            "count": 106
+            "accuracy": 0.057692307692307696,
+            "count": 52
           },
           "SS": {
-            "accuracy": 0.043478260869565216,
-            "count": 23
+            "accuracy": 0.14285714285714285,
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.09375,
-            "count": 160
+            "accuracy": 0.11235955056179775,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.05537459283387622,
-            "count": 307
+            "accuracy": 0.05844155844155844,
+            "count": 154
           }
         }
       },
       "add_S5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.08,
-        "n_examples": 100,
+        "digit_accuracy": 0.04857142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.05,
-            "count": 100
+            "accuracy": 0.1,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.14,
-            "count": 100
+            "accuracy": 0.04,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.11,
-            "count": 100
+            "accuracy": 0.12,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.065,
-            "count": 400
+            "accuracy": 0.02,
+            "count": 200
           }
         }
       },
       "add_S6": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.07571428571428572,
-        "n_examples": 100,
+        "digit_accuracy": 0.09428571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.03,
-            "count": 100
+            "accuracy": 0.14,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.15,
-            "count": 100
+            "accuracy": 0.18,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.07,
-            "count": 500
+            "accuracy": 0.068,
+            "count": 250
           }
         }
       },
       "add_random": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.1,
+        "digit_accuracy": 0.09714285714285714,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.09619686800894854,
-            "count": 447
+            "accuracy": 0.10672853828306264,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.1,
-            "count": 320
+            "accuracy": 0.10126582278481013,
+            "count": 316
           },
           "SS": {
-            "accuracy": 0.05357142857142857,
-            "count": 56
+            "accuracy": 0.05128205128205128,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.11342155009451796,
-            "count": 529
+            "accuracy": 0.09821428571428571,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.041666666666666664,
-            "count": 48
+            "accuracy": 0.018518518518518517,
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.11714285714285715,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.12,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 0.1,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.12,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.0,
+        "digit_accuracy": 0.09142857142857143,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 0.13,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 0.08,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.024096385542168676,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 0.0,
+            "count": 17
           }
         }
       },
       "add_C3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.09428571428571429,
-        "n_examples": 100,
+        "digit_accuracy": 0.11142857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.10666666666666667,
-            "count": 300
+            "accuracy": 0.14666666666666667,
+            "count": 150
           },
           "SC": {
-            "accuracy": 0.15,
-            "count": 100
+            "accuracy": 0.08,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.06735751295336788,
-            "count": 193
+            "accuracy": 0.11,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.056074766355140186,
-            "count": 107
+            "accuracy": 0.04,
+            "count": 50
           }
         }
       },
       "add_C4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.09428571428571429,
-        "n_examples": 100,
+        "digit_accuracy": 0.06571428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.05,
-            "count": 200
+            "accuracy": 0.09,
+            "count": 100
           },
           "SC": {
-            "accuracy": 0.14,
-            "count": 100
+            "accuracy": 0.06,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.1171875,
-            "count": 256
+            "accuracy": 0.045454545454545456,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.08333333333333333,
-            "count": 144
+            "accuracy": 0.07352941176470588,
+            "count": 68
           }
         }
       },
       "add_C5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.06,
-        "n_examples": 100,
+        "digit_accuracy": 0.07142857142857142,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.05,
-            "count": 100
+            "accuracy": 0.1,
+            "count": 50
           },
           "SC": {
-            "accuracy": 0.05,
-            "count": 100
+            "accuracy": 0.08,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.06862745098039216,
-            "count": 306
+            "accuracy": 0.0958904109589041,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.05670103092783505,
-            "count": 194
+            "accuracy": 0.019230769230769232,
+            "count": 104
           }
         }
       },
       "add_C6": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.09714285714285714,
-        "n_examples": 100,
+        "digit_accuracy": 0.08285714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
-            "accuracy": 0.15,
-            "count": 100
+            "accuracy": 0.06,
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.1092896174863388,
-            "count": 366
+            "accuracy": 0.10052910052910052,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.05555555555555555,
-            "count": 234
+            "accuracy": 0.06306306306306306,
+            "count": 111
           }
         }
       },
       "sub_M0": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.06857142857142857,
-        "n_examples": 100,
+        "digit_accuracy": 0.06285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.07154742096505824,
-            "count": 601
+            "accuracy": 0.0594059405940594,
+            "count": 303
           },
           "ME": {
-            "accuracy": 0.050505050505050504,
-            "count": 99
+            "accuracy": 0.0851063829787234,
+            "count": 47
           }
         }
       },
       "sub_M1": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.10714285714285714,
-        "n_examples": 100,
+        "digit_accuracy": 0.08285714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.08960573476702509,
-            "count": 279
+            "accuracy": 0.09219858156028368,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.10344827586206896,
-            "count": 145
+            "accuracy": 0.1111111111111111,
+            "count": 72
           },
           "ME": {
-            "accuracy": 0.08333333333333333,
-            "count": 24
+            "accuracy": 0.05555555555555555,
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.13095238095238096,
-            "count": 252
+            "accuracy": 0.058823529411764705,
+            "count": 119
           }
         }
       },
       "sub_M2": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.08571428571428572,
-        "n_examples": 100,
+        "digit_accuracy": 0.09428571428571429,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.06103286384976526,
-            "count": 213
+            "accuracy": 0.07142857142857142,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.11504424778761062,
-            "count": 113
+            "accuracy": 0.16981132075471697,
+            "count": 53
           },
           "ME": {
-            "accuracy": 0.09411764705882353,
-            "count": 85
+            "accuracy": 0.10638297872340426,
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.12154696132596685,
-            "count": 181
+            "accuracy": 0.09411764705882353,
+            "count": 85
           },
           "UD": {
-            "accuracy": 0.037037037037037035,
-            "count": 108
+            "accuracy": 0.05660377358490566,
+            "count": 53
           }
         }
       },
       "sub_M3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.09857142857142857,
-        "n_examples": 100,
+        "digit_accuracy": 0.07714285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.05027932960893855,
-            "count": 179
+            "accuracy": 0.061855670103092786,
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.1262135922330097,
-            "count": 103
+            "accuracy": 0.0392156862745098,
+            "count": 51
           },
           "ME": {
-            "accuracy": 0.10714285714285714,
-            "count": 56
+            "accuracy": 0.07407407407407407,
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.14093959731543623,
-            "count": 149
+            "accuracy": 0.10810810810810811,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.09389671361502347,
-            "count": 213
+            "accuracy": 0.0891089108910891,
+            "count": 101
           }
         }
       },
       "sub_M4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.06714285714285714,
-        "n_examples": 100,
+        "digit_accuracy": 0.06857142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.05,
-            "count": 200
+            "accuracy": 0.03,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.12,
-            "count": 100
+            "accuracy": 0.2,
+            "count": 50
           },
           "UB": {
             "accuracy": 0.08,
-            "count": 100
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.056666666666666664,
-            "count": 300
+            "accuracy": 0.04666666666666667,
+            "count": 150
           }
         }
       },
       "sub_M5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.08857142857142856,
-        "n_examples": 100,
+        "digit_accuracy": 0.054285714285714284,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.01,
-            "count": 100
+            "accuracy": 0.08,
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.1,
-            "count": 100
+            "accuracy": 0.02,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.13,
-            "count": 100
+            "accuracy": 0.12,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.095,
-            "count": 400
+            "accuracy": 0.04,
+            "count": 200
           }
         }
       },
       "sub_random": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.085,
+        "digit_accuracy": 0.09142857142857143,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.056666666666666664,
-            "count": 600
+            "accuracy": 0.07543859649122807,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.1198501872659176,
-            "count": 267
+            "accuracy": 0.11191335740072202,
+            "count": 277
           },
           "ME": {
-            "accuracy": 0.07547169811320754,
+            "accuracy": 0.018867924528301886,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.1070615034168565,
-            "count": 439
+            "accuracy": 0.10615711252653928,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.04878048780487805,
-            "count": 41
+            "accuracy": 0.10344827586206896,
+            "count": 29
           }
         }
       },
       "sub_B3": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.08857142857142856,
-        "n_examples": 100,
+        "digit_accuracy": 0.07714285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.07333333333333333,
-            "count": 300
+            "accuracy": 0.05333333333333334,
+            "count": 150
           },
           "MB": {
-            "accuracy": 0.08,
-            "count": 100
+            "accuracy": 0.12,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.14213197969543148,
-            "count": 197
+            "accuracy": 0.09900990099009901,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.038834951456310676,
-            "count": 103
+            "accuracy": 0.061224489795918366,
+            "count": 49
           }
         }
       },
       "sub_B4": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.08857142857142856,
-        "n_examples": 100,
+        "digit_accuracy": 0.08571428571428572,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.095,
-            "count": 200
+            "accuracy": 0.07,
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.09,
-            "count": 100
+            "accuracy": 0.04,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.10931174089068826,
-            "count": 247
+            "accuracy": 0.12396694214876033,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.0457516339869281,
-            "count": 153
+            "accuracy": 0.0759493670886076,
+            "count": 79
           }
         }
       },
       "sub_B5": {
         "full_accuracy": 0.0,
-        "digit_accuracy": 0.08571428571428572,
-        "n_examples": 100,
+        "digit_accuracy": 0.1,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.01,
-            "count": 100
+            "accuracy": 0.04,
+            "count": 50
           },
           "MB": {
-            "accuracy": 0.11,
-            "count": 100
+            "accuracy": 0.08,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.12416107382550336,
-            "count": 298
+            "accuracy": 0.1513157894736842,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.054455445544554455,
-            "count": 202
+            "accuracy": 0.061224489795918366,
+            "count": 98
           }
         }
       }
     },
     "summary": {
       "overall_accuracy": 0.0,
-      "digit_accuracy": 0.08678571428571429,
-      "total_examples": 2400,
-      "n_splits": 22
+      "digit_accuracy": 0.08504761904761905,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_eval": {
@@ -2239,525 +2281,567 @@
       "K": 1,
       "mode": "sorl",
       "n_digits": 6,
-      "n_per_split": 100
+      "n_per_split": 50
     },
     "splits": {
       "add_S0": {
-        "full_accuracy": 0.96,
-        "digit_accuracy": 0.9942857142857143,
-        "n_examples": 100,
+        "full_accuracy": 0.92,
+        "digit_accuracy": 0.9885714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9933884297520661,
-            "count": 605
+            "accuracy": 0.9864406779661017,
+            "count": 295
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 95
+            "count": 55
           }
         }
       },
       "add_S1": {
-        "full_accuracy": 0.96,
-        "digit_accuracy": 0.9942857142857143,
-        "n_examples": 100,
+        "full_accuracy": 0.92,
+        "digit_accuracy": 0.9885714285714285,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9950980392156863,
-            "count": 204
+            "accuracy": 0.9920634920634921,
+            "count": 126
           },
           "SC": {
-            "accuracy": 0.9940828402366864,
-            "count": 169
+            "accuracy": 1.0,
+            "count": 79
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 31
+            "count": 21
           },
           "UC": {
-            "accuracy": 0.9932432432432432,
-            "count": 296
+            "accuracy": 0.9758064516129032,
+            "count": 124
           }
         }
       },
       "add_S2": {
-        "full_accuracy": 0.87,
-        "digit_accuracy": 0.9771428571428571,
-        "n_examples": 100,
+        "full_accuracy": 0.76,
+        "digit_accuracy": 0.96,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9877300613496932,
-            "count": 163
+            "accuracy": 0.9866666666666667,
+            "count": 75
           },
           "SC": {
-            "accuracy": 0.9923076923076923,
-            "count": 130
+            "accuracy": 0.9838709677419355,
+            "count": 62
           },
           "SS": {
-            "accuracy": 0.9655172413793104,
-            "count": 87
+            "accuracy": 1.0,
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.9507389162561576,
-            "count": 203
+            "accuracy": 0.9009009009009009,
+            "count": 111
           },
           "US": {
-            "accuracy": 1.0,
-            "count": 117
+            "accuracy": 0.9841269841269841,
+            "count": 63
           }
         }
       },
       "add_S3": {
-        "full_accuracy": 0.67,
-        "digit_accuracy": 0.9342857142857143,
-        "n_examples": 100,
+        "full_accuracy": 0.7,
+        "digit_accuracy": 0.9457142857142857,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 121
+            "count": 60
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 121
+            "count": 57
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 49
+            "count": 19
           },
           "UC": {
-            "accuracy": 0.8225806451612904,
-            "count": 186
+            "accuracy": 0.8557692307692307,
+            "count": 104
           },
           "US": {
-            "accuracy": 0.9417040358744395,
-            "count": 223
+            "accuracy": 0.9636363636363636,
+            "count": 110
           }
         }
       },
       "add_S4": {
-        "full_accuracy": 0.48,
-        "digit_accuracy": 0.8728571428571429,
-        "n_examples": 100,
+        "full_accuracy": 0.32,
+        "digit_accuracy": 0.82,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 104
+            "count": 48
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 106
+            "count": 52
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 23
+            "count": 7
           },
           "UC": {
-            "accuracy": 0.71875,
-            "count": 160
+            "accuracy": 0.6292134831460674,
+            "count": 89
           },
           "US": {
-            "accuracy": 0.8566775244299675,
-            "count": 307
+            "accuracy": 0.8051948051948052,
+            "count": 154
           }
         }
       },
       "add_S5": {
-        "full_accuracy": 0.26,
-        "digit_accuracy": 0.6985714285714286,
-        "n_examples": 100,
+        "full_accuracy": 0.28,
+        "digit_accuracy": 0.7028571428571428,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.47,
-            "count": 100
+            "accuracy": 0.46,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.605,
-            "count": 400
+            "accuracy": 0.615,
+            "count": 200
           }
         }
       },
       "add_S6": {
-        "full_accuracy": 0.46,
-        "digit_accuracy": 0.71,
-        "n_examples": 100,
+        "full_accuracy": 0.34,
+        "digit_accuracy": 0.6371428571428571,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.5,
-            "count": 100
+            "accuracy": 0.36,
+            "count": 50
           },
           "US": {
-            "accuracy": 0.694,
-            "count": 500
+            "accuracy": 0.62,
+            "count": 250
           }
         }
       },
       "add_random": {
-        "full_accuracy": 0.915,
-        "digit_accuracy": 0.9871428571428571,
+        "full_accuracy": 0.95,
+        "digit_accuracy": 0.9928571428571429,
         "n_examples": 200,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9910514541387024,
-            "count": 447
+            "accuracy": 0.9930394431554525,
+            "count": 431
           },
           "SC": {
-            "accuracy": 0.996875,
-            "count": 320
+            "accuracy": 0.9968354430379747,
+            "count": 316
           },
           "SS": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 39
           },
           "UC": {
-            "accuracy": 0.9773156899810964,
-            "count": 529
+            "accuracy": 0.9892857142857143,
+            "count": 560
           },
           "US": {
-            "accuracy": 0.9791666666666666,
-            "count": 48
+            "accuracy": 1.0,
+            "count": 54
+          }
+        }
+      },
+      "add_C1": {
+        "full_accuracy": 0.9,
+        "digit_accuracy": 0.9857142857142858,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 250
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.9,
+            "count": 50
+          }
+        }
+      },
+      "add_C2": {
+        "full_accuracy": 0.82,
+        "digit_accuracy": 0.96,
+        "n_examples": 50,
+        "per_subtask": {
+          "SA": {
+            "accuracy": 1.0,
+            "count": 200
+          },
+          "SC": {
+            "accuracy": 1.0,
+            "count": 50
+          },
+          "UC": {
+            "accuracy": 0.9036144578313253,
+            "count": 83
+          },
+          "US": {
+            "accuracy": 0.6470588235294118,
+            "count": 17
           }
         }
       },
       "add_C3": {
-        "full_accuracy": 0.64,
-        "digit_accuracy": 0.9357142857142857,
-        "n_examples": 100,
+        "full_accuracy": 0.58,
+        "digit_accuracy": 0.9114285714285715,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
-            "accuracy": 0.9966666666666667,
-            "count": 300
+            "accuracy": 1.0,
+            "count": 150
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.844559585492228,
-            "count": 193
+            "accuracy": 0.82,
+            "count": 100
           },
           "US": {
-            "accuracy": 0.8691588785046729,
-            "count": 107
+            "accuracy": 0.74,
+            "count": 50
           }
         }
       },
       "add_C4": {
-        "full_accuracy": 0.64,
-        "digit_accuracy": 0.9257142857142857,
-        "n_examples": 100,
+        "full_accuracy": 0.66,
+        "digit_accuracy": 0.9285714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.875,
-            "count": 256
+            "accuracy": 0.8636363636363636,
+            "count": 132
           },
           "US": {
-            "accuracy": 0.8611111111111112,
-            "count": 144
+            "accuracy": 0.8970588235294118,
+            "count": 68
           }
         }
       },
       "add_C5": {
-        "full_accuracy": 0.62,
-        "digit_accuracy": 0.9114285714285715,
-        "n_examples": 100,
+        "full_accuracy": 0.56,
+        "digit_accuracy": 0.9028571428571428,
+        "n_examples": 50,
         "per_subtask": {
           "SA": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.8660130718954249,
-            "count": 306
+            "accuracy": 0.8493150684931506,
+            "count": 146
           },
           "US": {
-            "accuracy": 0.8917525773195877,
-            "count": 194
+            "accuracy": 0.8846153846153846,
+            "count": 104
           }
         }
       },
       "add_C6": {
-        "full_accuracy": 0.7,
-        "digit_accuracy": 0.9428571428571428,
-        "n_examples": 100,
+        "full_accuracy": 0.62,
+        "digit_accuracy": 0.9342857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "SC": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UC": {
-            "accuracy": 0.9180327868852459,
-            "count": 366
+            "accuracy": 0.91005291005291,
+            "count": 189
           },
           "US": {
-            "accuracy": 0.9572649572649573,
-            "count": 234
+            "accuracy": 0.9459459459459459,
+            "count": 111
           }
         }
       },
       "sub_M0": {
-        "full_accuracy": 0.94,
-        "digit_accuracy": 0.9914285714285714,
-        "n_examples": 100,
+        "full_accuracy": 0.96,
+        "digit_accuracy": 0.9942857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9900166389351082,
-            "count": 601
+            "accuracy": 0.9933993399339934,
+            "count": 303
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 99
+            "count": 47
           }
         }
       },
       "sub_M1": {
-        "full_accuracy": 0.92,
-        "digit_accuracy": 0.9885714285714285,
-        "n_examples": 100,
+        "full_accuracy": 0.94,
+        "digit_accuracy": 0.9914285714285714,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.982078853046595,
-            "count": 279
+            "accuracy": 0.9929078014184397,
+            "count": 141
           },
           "MB": {
-            "accuracy": 0.993103448275862,
-            "count": 145
+            "accuracy": 0.9861111111111112,
+            "count": 72
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 24
+            "count": 18
           },
           "UB": {
-            "accuracy": 0.9920634920634921,
-            "count": 252
+            "accuracy": 0.9915966386554622,
+            "count": 119
           }
         }
       },
       "sub_M2": {
-        "full_accuracy": 0.67,
+        "full_accuracy": 0.7,
         "digit_accuracy": 0.9514285714285714,
-        "n_examples": 100,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9906103286384976,
-            "count": 213
+            "accuracy": 0.9910714285714286,
+            "count": 112
           },
           "MB": {
-            "accuracy": 0.9911504424778761,
-            "count": 113
+            "accuracy": 0.9811320754716981,
+            "count": 53
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 85
+            "count": 47
           },
           "UB": {
-            "accuracy": 0.8287292817679558,
-            "count": 181
+            "accuracy": 0.8352941176470589,
+            "count": 85
           },
           "UD": {
-            "accuracy": 1.0,
-            "count": 108
+            "accuracy": 0.9811320754716981,
+            "count": 53
           }
         }
       },
       "sub_M3": {
-        "full_accuracy": 0.26,
-        "digit_accuracy": 0.8557142857142858,
-        "n_examples": 100,
+        "full_accuracy": 0.24,
+        "digit_accuracy": 0.8542857142857143,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 179
+            "count": 97
           },
           "MB": {
-            "accuracy": 0.9805825242718447,
-            "count": 103
+            "accuracy": 0.9607843137254902,
+            "count": 51
           },
           "ME": {
             "accuracy": 1.0,
-            "count": 56
+            "count": 27
           },
           "UB": {
-            "accuracy": 0.5704697986577181,
-            "count": 149
+            "accuracy": 0.5675675675675675,
+            "count": 74
           },
           "UD": {
-            "accuracy": 0.8356807511737089,
-            "count": 213
+            "accuracy": 0.8316831683168316,
+            "count": 101
           }
         }
       },
       "sub_M4": {
-        "full_accuracy": 0.06,
-        "digit_accuracy": 0.7242857142857143,
-        "n_examples": 100,
+        "full_accuracy": 0.08,
+        "digit_accuracy": 0.7257142857142858,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.31,
-            "count": 100
+            "accuracy": 0.28,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.5866666666666667,
-            "count": 300
+            "accuracy": 0.6,
+            "count": 150
           }
         }
       },
       "sub_M5": {
-        "full_accuracy": 0.01,
-        "digit_accuracy": 0.5328571428571428,
-        "n_examples": 100,
+        "full_accuracy": 0.02,
+        "digit_accuracy": 0.5428571428571428,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.2,
-            "count": 100
+            "accuracy": 0.36,
+            "count": 50
           },
           "UD": {
-            "accuracy": 0.3825,
-            "count": 400
+            "accuracy": 0.36,
+            "count": 200
           }
         }
       },
       "sub_random": {
-        "full_accuracy": 0.905,
-        "digit_accuracy": 0.9857142857142858,
+        "full_accuracy": 0.895,
+        "digit_accuracy": 0.9821428571428571,
         "n_examples": 200,
         "per_subtask": {
           "MD": {
-            "accuracy": 0.9983333333333333,
-            "count": 600
+            "accuracy": 0.9859649122807017,
+            "count": 570
           },
           "MB": {
-            "accuracy": 0.9925093632958801,
-            "count": 267
+            "accuracy": 0.9927797833935018,
+            "count": 277
           },
           "ME": {
             "accuracy": 1.0,
             "count": 53
           },
           "UB": {
-            "accuracy": 0.9635535307517085,
-            "count": 439
+            "accuracy": 0.970276008492569,
+            "count": 471
           },
           "UD": {
-            "accuracy": 0.975609756097561,
-            "count": 41
+            "accuracy": 0.9655172413793104,
+            "count": 29
           }
         }
       },
       "sub_B3": {
-        "full_accuracy": 0.7,
-        "digit_accuracy": 0.9457142857142857,
-        "n_examples": 100,
+        "full_accuracy": 0.72,
+        "digit_accuracy": 0.9485714285714286,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
-            "accuracy": 1.0,
-            "count": 300
+            "accuracy": 0.9933333333333333,
+            "count": 150
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.8629441624365483,
-            "count": 197
+            "accuracy": 0.8910891089108911,
+            "count": 101
           },
           "UD": {
-            "accuracy": 0.8932038834951457,
-            "count": 103
+            "accuracy": 0.8775510204081632,
+            "count": 49
           }
         }
       },
       "sub_B4": {
-        "full_accuracy": 0.54,
-        "digit_accuracy": 0.9057142857142857,
-        "n_examples": 100,
+        "full_accuracy": 0.38,
+        "digit_accuracy": 0.8657142857142858,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 200
+            "count": 100
           },
           "MB": {
-            "accuracy": 0.99,
-            "count": 100
+            "accuracy": 0.96,
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.8380566801619433,
-            "count": 247
+            "accuracy": 0.768595041322314,
+            "count": 121
           },
           "UD": {
-            "accuracy": 0.8366013071895425,
-            "count": 153
+            "accuracy": 0.7848101265822784,
+            "count": 79
           }
         }
       },
       "sub_B5": {
-        "full_accuracy": 0.43,
-        "digit_accuracy": 0.8685714285714285,
-        "n_examples": 100,
+        "full_accuracy": 0.4,
+        "digit_accuracy": 0.8657142857142858,
+        "n_examples": 50,
         "per_subtask": {
           "MD": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "MB": {
             "accuracy": 1.0,
-            "count": 100
+            "count": 50
           },
           "UB": {
-            "accuracy": 0.8120805369127517,
-            "count": 298
+            "accuracy": 0.8223684210526315,
+            "count": 152
           },
           "UD": {
-            "accuracy": 0.8217821782178217,
-            "count": 202
+            "accuracy": 0.7959183673469388,
+            "count": 98
           }
         }
       }
     },
     "summary": {
-      "overall_accuracy": 0.6429166666666667,
-      "digit_accuracy": 0.9002380952380953,
-      "total_examples": 2400,
-      "n_splits": 22
+      "overall_accuracy": 0.6726666666666666,
+      "digit_accuracy": 0.9100952380952381,
+      "total_examples": 1500,
+      "n_splits": 24
     }
   },
   "sorl_overall_accuracy": 0.6429166666666667,