Bagratuni commited on
Commit
c17f99a
·
verified ·
1 Parent(s): 151453b

add microsoft harrier model result (270m, 596m variants)

Browse files
Files changed (1) hide show
  1. model_results.json +108 -0
model_results.json CHANGED
@@ -1,4 +1,112 @@
1
  [
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
  {
3
  "model_name": "Alibaba-NLP/gte-multilingual-base",
4
  "model_url": "https://huggingface.co/Alibaba-NLP/gte-multilingual-base",
 
1
  [
2
+ {
3
+ "model_name": "microsoft/harrier-oss-v1-270m",
4
+ "model_url": "https://huggingface.co/microsoft/harrier-oss-v1-270m",
5
+ "results": {
6
+ "mteb_avg": 0.6526,
7
+ "mteb_detailed": {
8
+ "FloresBitextMining_devtest": 0.1636,
9
+ "NTREXBitextMining_test": 0.939,
10
+ "Tatoeba_test": 0.9093,
11
+ "MassiveIntentClassification_test": 0.534,
12
+ "MassiveScenarioClassification_test": 0.5891,
13
+ "SIB200Classification_test": 0.7108,
14
+ "SIB200ClusteringS2S_test": 0.3309,
15
+ "ArmenianParaphrasePC_test": 0.9281,
16
+ "BelebeleRetrieval_test": 0.7683
17
+ },
18
+ "sts_spearman": 0.7092,
19
+ "sts_detailed": {
20
+ "Pearson_correlation": 0.6992,
21
+ "Spearman_correlation": 0.7092
22
+ },
23
+ "retrieval_top20": 0.7011,
24
+ "retrieval_detailed": {
25
+ "top1 within document": 0.4,
26
+ "top3 within document": 0.6,
27
+ "top5 within document": 0.74,
28
+ "top20 group mean macro": 0.8179,
29
+ "top20 all": 0.7011
30
+ },
31
+ "msmarco_top10": 0.724,
32
+ "msmarco_detailed": {
33
+ "reranking_mrr": 0.5237,
34
+ "retrieval_mrr": 0.3992,
35
+ "retrieval_top5_accuracy": 0.6003,
36
+ "retrieval_top10_accuracy": 0.724
37
+ },
38
+ "retrieval_translit_top20": 0.2011,
39
+ "retrieval_translit_detailed": {
40
+ "top1 within document": 0.19,
41
+ "top3 within document": 0.36,
42
+ "top5 within document": 0.44,
43
+ "top20 group mean macro": 0.4227,
44
+ "top20 all": 0.2011
45
+ },
46
+ "msmarco_translit_top10": 0.1719,
47
+ "msmarco_translit_detailed": {
48
+ "reranking_mrr": 0.367,
49
+ "retrieval_mrr": 0.0828,
50
+ "retrieval_top5_accuracy": 0.1267,
51
+ "retrieval_top10_accuracy": 0.1719
52
+ }
53
+ },
54
+ "model_size": "270M"
55
+ },
56
+ {
57
+ "model_name": "microsoft/harrier-oss-v1-0.6b",
58
+ "model_url": "https://huggingface.co/microsoft/harrier-oss-v1-0.6b",
59
+ "results": {
60
+ "mteb_avg": 0.6533,
61
+ "mteb_detailed": {
62
+ "FloresBitextMining_devtest": 0.2497,
63
+ "NTREXBitextMining_test": 0.9432,
64
+ "Tatoeba_test": 0.8939,
65
+ "MassiveIntentClassification_test": 0.5296,
66
+ "MassiveScenarioClassification_test": 0.5976,
67
+ "SIB200Classification_test": 0.6907,
68
+ "SIB200ClusteringS2S_test": 0.2729,
69
+ "ArmenianParaphrasePC_test": 0.9305,
70
+ "BelebeleRetrieval_test": 0.7711
71
+ },
72
+ "sts_spearman": 0.718,
73
+ "sts_detailed": {
74
+ "Pearson_correlation": 0.7053,
75
+ "Spearman_correlation": 0.718
76
+ },
77
+ "retrieval_top20": 0.6087,
78
+ "retrieval_detailed": {
79
+ "top1 within document": 0.38,
80
+ "top3 within document": 0.55,
81
+ "top5 within document": 0.7,
82
+ "top20 group mean macro": 0.7686,
83
+ "top20 all": 0.6087
84
+ },
85
+ "msmarco_top10": 0.7771,
86
+ "msmarco_detailed": {
87
+ "reranking_mrr": 0.5506,
88
+ "retrieval_mrr": 0.4395,
89
+ "retrieval_top5_accuracy": 0.6544,
90
+ "retrieval_top10_accuracy": 0.7771
91
+ },
92
+ "retrieval_translit_top20": 0.2283,
93
+ "retrieval_translit_detailed": {
94
+ "top1 within document": 0.16,
95
+ "top3 within document": 0.36,
96
+ "top5 within document": 0.49,
97
+ "top20 group mean macro": 0.4179,
98
+ "top20 all": 0.2283
99
+ },
100
+ "msmarco_translit_top10": 0.2612,
101
+ "msmarco_translit_detailed": {
102
+ "reranking_mrr": 0.4011,
103
+ "retrieval_mrr": 0.1312,
104
+ "retrieval_top5_accuracy": 0.1996,
105
+ "retrieval_top10_accuracy": 0.2612
106
+ }
107
+ },
108
+ "model_size": "596M"
109
+ },
110
  {
111
  "model_name": "Alibaba-NLP/gte-multilingual-base",
112
  "model_url": "https://huggingface.co/Alibaba-NLP/gte-multilingual-base",