Create benchmarks.md
Browse files- benchmarks.md +79 -0
benchmarks.md
ADDED
|
@@ -0,0 +1,79 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
| Tasks |Version|Filter|n-shot| Metric | | Value | |Stderr|
|
| 2 |
+
|------------------------------------------------------------|------:|------|-----:|---------------|---|---------:|---|------|
|
| 3 |
+
|blimp | 2|none | 0|acc |↑ | 0.5354|± |0.0017|
|
| 4 |
+
| - blimp_adjunct_island | 1|none | 0|acc |↑ | 0.5980|± |0.0155|
|
| 5 |
+
| - blimp_anaphor_gender_agreement | 1|none | 0|acc |↑ | 0.3130|± |0.0147|
|
| 6 |
+
| - blimp_anaphor_number_agreement | 1|none | 0|acc |↑ | 0.5090|± |0.0158|
|
| 7 |
+
| - blimp_animate_subject_passive | 1|none | 0|acc |↑ | 0.5750|± |0.0156|
|
| 8 |
+
| - blimp_animate_subject_trans | 1|none | 0|acc |↑ | 0.7470|± |0.0138|
|
| 9 |
+
| - blimp_causative | 1|none | 0|acc |↑ | 0.4810|± |0.0158|
|
| 10 |
+
| - blimp_complex_NP_island | 1|none | 0|acc |↑ | 0.4880|± |0.0158|
|
| 11 |
+
| - blimp_coordinate_structure_constraint_complex_left_branch| 1|none | 0|acc |↑ | 0.1420|± |0.0110|
|
| 12 |
+
| - blimp_coordinate_structure_constraint_object_extraction | 1|none | 0|acc |↑ | 0.5820|± |0.0156|
|
| 13 |
+
| - blimp_determiner_noun_agreement_1 | 1|none | 0|acc |↑ | 0.6580|± |0.0150|
|
| 14 |
+
| - blimp_determiner_noun_agreement_2 | 1|none | 0|acc |↑ | 0.6320|± |0.0153|
|
| 15 |
+
| - blimp_determiner_noun_agreement_irregular_1 | 1|none | 0|acc |↑ | 0.5150|± |0.0158|
|
| 16 |
+
| - blimp_determiner_noun_agreement_irregular_2 | 1|none | 0|acc |↑ | 0.6980|± |0.0145|
|
| 17 |
+
| - blimp_determiner_noun_agreement_with_adj_2 | 1|none | 0|acc |↑ | 0.5780|± |0.0156|
|
| 18 |
+
| - blimp_determiner_noun_agreement_with_adj_irregular_1 | 1|none | 0|acc |↑ | 0.4220|± |0.0156|
|
| 19 |
+
| - blimp_determiner_noun_agreement_with_adj_irregular_2 | 1|none | 0|acc |↑ | 0.5170|± |0.0158|
|
| 20 |
+
| - blimp_determiner_noun_agreement_with_adjective_1 | 1|none | 0|acc |↑ | 0.6070|± |0.0155|
|
| 21 |
+
| - blimp_distractor_agreement_relational_noun | 1|none | 0|acc |↑ | 0.3120|± |0.0147|
|
| 22 |
+
| - blimp_distractor_agreement_relative_clause | 1|none | 0|acc |↑ | 0.3110|± |0.0146|
|
| 23 |
+
| - blimp_drop_argument | 1|none | 0|acc |↑ | 0.7270|± |0.0141|
|
| 24 |
+
| - blimp_ellipsis_n_bar_1 | 1|none | 0|acc |↑ | 0.2180|± |0.0131|
|
| 25 |
+
| - blimp_ellipsis_n_bar_2 | 1|none | 0|acc |↑ | 0.3480|± |0.0151|
|
| 26 |
+
| - blimp_existential_there_object_raising | 1|none | 0|acc |↑ | 0.6860|± |0.0147|
|
| 27 |
+
| - blimp_existential_there_quantifiers_1 | 1|none | 0|acc |↑ | 0.8100|± |0.0124|
|
| 28 |
+
| - blimp_existential_there_quantifiers_2 | 1|none | 0|acc |↑ | 0.2950|± |0.0144|
|
| 29 |
+
| - blimp_existential_there_subject_raising | 1|none | 0|acc |↑ | 0.6880|± |0.0147|
|
| 30 |
+
| - blimp_expletive_it_object_raising | 1|none | 0|acc |↑ | 0.6570|± |0.0150|
|
| 31 |
+
| - blimp_inchoative | 1|none | 0|acc |↑ | 0.3850|± |0.0154|
|
| 32 |
+
| - blimp_intransitive | 1|none | 0|acc |↑ | 0.5170|± |0.0158|
|
| 33 |
+
| - blimp_irregular_past_participle_adjectives | 1|none | 0|acc |↑ | 0.6620|± |0.0150|
|
| 34 |
+
| - blimp_irregular_past_participle_verbs | 1|none | 0|acc |↑ | 0.5050|± |0.0158|
|
| 35 |
+
| - blimp_irregular_plural_subject_verb_agreement_1 | 1|none | 0|acc |↑ | 0.5880|± |0.0156|
|
| 36 |
+
| - blimp_irregular_plural_subject_verb_agreement_2 | 1|none | 0|acc |↑ | 0.5860|± |0.0156|
|
| 37 |
+
| - blimp_left_branch_island_echo_question | 1|none | 0|acc |↑ | 0.9020|± |0.0094|
|
| 38 |
+
| - blimp_left_branch_island_simple_question | 1|none | 0|acc |↑ | 0.2310|± |0.0133|
|
| 39 |
+
| - blimp_matrix_question_npi_licensor_present | 1|none | 0|acc |↑ | 0.0380|± |0.0060|
|
| 40 |
+
| - blimp_npi_present_1 | 1|none | 0|acc |↑ | 0.6520|± |0.0151|
|
| 41 |
+
| - blimp_npi_present_2 | 1|none | 0|acc |↑ | 0.6390|± |0.0152|
|
| 42 |
+
| - blimp_only_npi_licensor_present | 1|none | 0|acc |↑ | 0.0400|± |0.0062|
|
| 43 |
+
| - blimp_only_npi_scope | 1|none | 0|acc |↑ | 0.0020|± |0.0014|
|
| 44 |
+
| - blimp_passive_1 | 1|none | 0|acc |↑ | 0.6520|± |0.0151|
|
| 45 |
+
| - blimp_passive_2 | 1|none | 0|acc |↑ | 0.6280|± |0.0153|
|
| 46 |
+
| - blimp_principle_A_c_command | 1|none | 0|acc |↑ | 0.6890|± |0.0146|
|
| 47 |
+
| - blimp_principle_A_case_1 | 1|none | 0|acc |↑ | 0.9990|± |0.0010|
|
| 48 |
+
| - blimp_principle_A_case_2 | 1|none | 0|acc |↑ | 0.4450|± |0.0157|
|
| 49 |
+
| - blimp_principle_A_domain_1 | 1|none | 0|acc |↑ | 0.8820|± |0.0102|
|
| 50 |
+
| - blimp_principle_A_domain_2 | 1|none | 0|acc |↑ | 0.5450|± |0.0158|
|
| 51 |
+
| - blimp_principle_A_domain_3 | 1|none | 0|acc |↑ | 0.4690|± |0.0158|
|
| 52 |
+
| - blimp_principle_A_reconstruction | 1|none | 0|acc |↑ | 0.3830|± |0.0154|
|
| 53 |
+
| - blimp_regular_plural_subject_verb_agreement_1 | 1|none | 0|acc |↑ | 0.6890|± |0.0146|
|
| 54 |
+
| - blimp_regular_plural_subject_verb_agreement_2 | 1|none | 0|acc |↑ | 0.5760|± |0.0156|
|
| 55 |
+
| - blimp_sentential_negation_npi_licensor_present | 1|none | 0|acc |↑ | 0.9990|± |0.0010|
|
| 56 |
+
| - blimp_sentential_negation_npi_scope | 1|none | 0|acc |↑ | 0.4590|± |0.0158|
|
| 57 |
+
| - blimp_sentential_subject_island | 1|none | 0|acc |↑ | 0.2760|± |0.0141|
|
| 58 |
+
| - blimp_superlative_quantifiers_1 | 1|none | 0|acc |↑ | 0.3040|± |0.0146|
|
| 59 |
+
| - blimp_superlative_quantifiers_2 | 1|none | 0|acc |↑ | 0.3620|± |0.0152|
|
| 60 |
+
| - blimp_tough_vs_raising_1 | 1|none | 0|acc |↑ | 0.3310|± |0.0149|
|
| 61 |
+
| - blimp_tough_vs_raising_2 | 1|none | 0|acc |↑ | 0.6970|± |0.0145|
|
| 62 |
+
| - blimp_transitive | 1|none | 0|acc |↑ | 0.6560|± |0.0150|
|
| 63 |
+
| - blimp_wh_island | 1|none | 0|acc |↑ | 0.5110|± |0.0158|
|
| 64 |
+
| - blimp_wh_questions_object_gap | 1|none | 0|acc |↑ | 0.6180|± |0.0154|
|
| 65 |
+
| - blimp_wh_questions_subject_gap | 1|none | 0|acc |↑ | 0.9480|± |0.0070|
|
| 66 |
+
| - blimp_wh_questions_subject_gap_long_distance | 1|none | 0|acc |↑ | 0.8930|± |0.0098|
|
| 67 |
+
| - blimp_wh_vs_that_no_gap | 1|none | 0|acc |↑ | 0.9960|± |0.0020|
|
| 68 |
+
| - blimp_wh_vs_that_no_gap_long_distance | 1|none | 0|acc |↑ | 0.9910|± |0.0030|
|
| 69 |
+
| - blimp_wh_vs_that_with_gap | 1|none | 0|acc |↑ | 0.0090|± |0.0030|
|
| 70 |
+
| - blimp_wh_vs_that_with_gap_long_distance | 1|none | 0|acc |↑ | 0.0040|± |0.0020|
|
| 71 |
+
|arc_easy | 1|none | 0|acc |↑ | 0.2677|± |0.0091|
|
| 72 |
+
| | |none | 0|acc_norm |↑ | 0.2841|± |0.0093|
|
| 73 |
+
|wikitext | 2|none | 0|bits_per_byte |↓ | 2.9624|± | N/A|
|
| 74 |
+
| | |none | 0|byte_perplexity|↓ | 7.7940|± | N/A|
|
| 75 |
+
| | |none | 0|word_perplexity|↓ |58699.2441|± | N/A|
|
| 76 |
+
|
| 77 |
+
|Groups|Version|Filter|n-shot|Metric| |Value | |Stderr|
|
| 78 |
+
|------|------:|------|-----:|------|---|-----:|---|-----:|
|
| 79 |
+
|blimp | 2|none | 0|acc |↑ |0.5354|± |0.0017|
|