diff --git "a/scores/LamRA-Ret-Qwen2.5VL-7b.json" "b/scores/LamRA-Ret-Qwen2.5VL-7b.json" --- "a/scores/LamRA-Ret-Qwen2.5VL-7b.json" +++ "b/scores/LamRA-Ret-Qwen2.5VL-7b.json" @@ -9,1264 +9,1264 @@ "metrics": { "image": { "ImageNet-1K": { - "hit@1": 0.589, - "hit@5": 0.819, - "hit@10": 0.889, - "ndcg_linear@1": 0.589, - "ndcg_linear@5": 0.7149684169244714, - "ndcg_linear@10": 0.7376117600535983, - "ndcg_exponential@1": 0.589, - "ndcg_exponential@5": 0.7149684169244714, - "ndcg_exponential@10": 0.7376117600535983, - "precision@1": 0.589, - "precision@5": 0.1638, - "precision@10": 0.0889, - "recall@1": 0.589, - "recall@5": 0.819, - "recall@10": 0.889, - "f1@1": 0.589, - "f1@5": 0.273, - "f1@10": 0.16163636363636363, - "map@1": 0.589, - "map@5": 0.68, - "map@10": 0.689347619047619, - "mrr@1": 0.589, - "mrr@5": 0.68, - "mrr@10": 0.689347619047619, + "hit@1": 0.592, + "hit@5": 0.828, + "hit@10": 0.897, + "ndcg_linear@1": 0.592, + "ndcg_linear@5": 0.7199949989383112, + "ndcg_linear@10": 0.7422507294326354, + "ndcg_exponential@1": 0.592, + "ndcg_exponential@5": 0.7199949989383112, + "ndcg_exponential@10": 0.7422507294326354, + "precision@1": 0.592, + "precision@5": 0.16560000000000002, + "precision@10": 0.08970000000000002, + "recall@1": 0.592, + "recall@5": 0.828, + "recall@10": 0.897, + "f1@1": 0.592, + "f1@5": 0.276, + "f1@10": 0.16309090909090906, + "map@1": 0.592, + "map@5": 0.6838166666666666, + "map@10": 0.6929666666666667, + "mrr@1": 0.592, + "mrr@5": 0.6838166666666666, + "mrr@10": 0.6929666666666667, "num_pred": 1000, "num_data": 1000 }, "N24News": { - "hit@1": 0.298, - "hit@5": 0.615, - "hit@10": 0.823, - "ndcg_linear@1": 0.298, - "ndcg_linear@5": 0.4631110593542077, - "ndcg_linear@10": 0.5303223739475101, - "ndcg_exponential@1": 0.298, - "ndcg_exponential@5": 0.4631110593542077, - "ndcg_exponential@10": 0.5303223739475101, - "precision@1": 0.298, - "precision@5": 0.123, - "precision@10": 0.0823, - "recall@1": 0.298, - "recall@5": 0.615, - "recall@10": 0.823, - "f1@1": 0.298, - "f1@5": 0.20500000000000004, - "f1@10": 0.14963636363636362, - "map@1": 0.298, - "map@5": 0.4129166666666667, - "map@10": 0.44063373015873014, - "mrr@1": 0.298, - "mrr@5": 0.4129166666666667, - "mrr@10": 0.44063373015873014, + "hit@1": 0.304, + "hit@5": 0.623, + "hit@10": 0.818, + "ndcg_linear@1": 0.304, + "ndcg_linear@5": 0.4698574584762916, + "ndcg_linear@10": 0.5330951747261412, + "ndcg_exponential@1": 0.304, + "ndcg_exponential@5": 0.4698574584762916, + "ndcg_exponential@10": 0.5330951747261412, + "precision@1": 0.304, + "precision@5": 0.12459999999999999, + "precision@10": 0.08180000000000001, + "recall@1": 0.304, + "recall@5": 0.623, + "recall@10": 0.818, + "f1@1": 0.304, + "f1@5": 0.20766666666666675, + "f1@10": 0.14872727272727274, + "map@1": 0.304, + "map@5": 0.41923333333333335, + "map@10": 0.44545317460317463, + "mrr@1": 0.304, + "mrr@5": 0.41923333333333335, + "mrr@10": 0.44545317460317463, "num_pred": 1000, "num_data": 1000 }, "HatefulMemes": { - "hit@1": 0.513, + "hit@1": 0.502, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.513, - "ndcg_linear@5": 0.8202627899893, - "ndcg_linear@10": 0.8202627899893, - "ndcg_exponential@1": 0.513, - "ndcg_exponential@5": 0.8202627899893, - "ndcg_exponential@10": 0.8202627899893, - "precision@1": 0.513, + "ndcg_linear@1": 0.502, + "ndcg_linear@5": 0.816203017278586, + "ndcg_linear@10": 0.816203017278586, + "ndcg_exponential@1": 0.502, + "ndcg_exponential@5": 0.816203017278586, + "ndcg_exponential@10": 0.816203017278586, + "precision@1": 0.502, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.513, + "recall@1": 0.502, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.513, + "f1@1": 0.502, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.513, - "map@5": 0.7565, - "map@10": 0.7565, - "mrr@1": 0.513, - "mrr@5": 0.7565, - "mrr@10": 0.7565, + "map@1": 0.502, + "map@5": 0.751, + "map@10": 0.751, + "mrr@1": 0.502, + "mrr@5": 0.751, + "mrr@10": 0.751, "num_pred": 1000, "num_data": 1000 }, "VOC2007": { - "hit@1": 0.787, - "hit@5": 0.97, - "hit@10": 0.989, - "ndcg_linear@1": 0.787, - "ndcg_linear@5": 0.8883866275027872, - "ndcg_linear@10": 0.8944932963244013, - "ndcg_exponential@1": 0.787, - "ndcg_exponential@5": 0.8883866275027872, - "ndcg_exponential@10": 0.8944932963244013, - "precision@1": 0.787, - "precision@5": 0.19400000000000003, - "precision@10": 0.09890000000000002, - "recall@1": 0.787, - "recall@5": 0.97, - "recall@10": 0.989, - "f1@1": 0.787, - "f1@5": 0.32333333333333336, - "f1@10": 0.17981818181818182, - "map@1": 0.787, - "map@5": 0.8608166666666666, - "map@10": 0.8633138888888888, - "mrr@1": 0.787, - "mrr@5": 0.8608166666666666, - "mrr@10": 0.8633138888888888, + "hit@1": 0.782, + "hit@5": 0.968, + "hit@10": 0.985, + "ndcg_linear@1": 0.782, + "ndcg_linear@5": 0.8851668110696811, + "ndcg_linear@10": 0.8907724107726365, + "ndcg_exponential@1": 0.782, + "ndcg_exponential@5": 0.8851668110696811, + "ndcg_exponential@10": 0.8907724107726365, + "precision@1": 0.782, + "precision@5": 0.19360000000000002, + "precision@10": 0.09850000000000003, + "recall@1": 0.782, + "recall@5": 0.968, + "recall@10": 0.985, + "f1@1": 0.782, + "f1@5": 0.3226666666666667, + "f1@10": 0.17909090909090908, + "map@1": 0.782, + "map@5": 0.8571666666666667, + "map@10": 0.8595452380952381, + "mrr@1": 0.782, + "mrr@5": 0.8571666666666667, + "mrr@10": 0.8595452380952381, "num_pred": 1000, "num_data": 1000 }, "SUN397": { - "hit@1": 0.665, - "hit@5": 0.904, - "hit@10": 0.952, - "ndcg_linear@1": 0.665, - "ndcg_linear@5": 0.7979081033899503, - "ndcg_linear@10": 0.8134286390471887, - "ndcg_exponential@1": 0.665, - "ndcg_exponential@5": 0.7979081033899503, - "ndcg_exponential@10": 0.8134286390471887, - "precision@1": 0.665, - "precision@5": 0.18080000000000004, - "precision@10": 0.09520000000000002, - "recall@1": 0.665, - "recall@5": 0.904, - "recall@10": 0.952, - "f1@1": 0.665, - "f1@5": 0.3013333333333334, - "f1@10": 0.17309090909090907, - "map@1": 0.665, - "map@5": 0.7620333333333333, - "map@10": 0.768438492063492, - "mrr@1": 0.665, - "mrr@5": 0.7620333333333333, - "mrr@10": 0.768438492063492, + "hit@1": 0.649, + "hit@5": 0.888, + "hit@10": 0.935, + "ndcg_linear@1": 0.649, + "ndcg_linear@5": 0.7822777412598195, + "ndcg_linear@10": 0.7973927958257072, + "ndcg_exponential@1": 0.649, + "ndcg_exponential@5": 0.7822777412598195, + "ndcg_exponential@10": 0.7973927958257072, + "precision@1": 0.649, + "precision@5": 0.17760000000000004, + "precision@10": 0.09350000000000003, + "recall@1": 0.649, + "recall@5": 0.888, + "recall@10": 0.935, + "f1@1": 0.649, + "f1@5": 0.296, + "f1@10": 0.17, + "map@1": 0.649, + "map@5": 0.7465333333333333, + "map@10": 0.7527230158730159, + "mrr@1": 0.649, + "mrr@5": 0.7465333333333333, + "mrr@10": 0.7527230158730159, "num_pred": 1000, "num_data": 1000 }, "Place365": { - "hit@1": 0.374, - "hit@5": 0.642, - "hit@10": 0.741, - "ndcg_linear@1": 0.374, - "ndcg_linear@5": 0.514784678286875, - "ndcg_linear@10": 0.5465467859742046, - "ndcg_exponential@1": 0.374, - "ndcg_exponential@5": 0.514784678286875, - "ndcg_exponential@10": 0.5465467859742046, - "precision@1": 0.374, - "precision@5": 0.12840000000000001, - "precision@10": 0.0741, - "recall@1": 0.374, - "recall@5": 0.642, - "recall@10": 0.741, - "f1@1": 0.374, - "f1@5": 0.21400000000000005, - "f1@10": 0.13472727272727275, - "map@1": 0.374, - "map@5": 0.47264999999999996, - "map@10": 0.4856075396825397, - "mrr@1": 0.374, - "mrr@5": 0.47264999999999996, - "mrr@10": 0.4856075396825397, + "hit@1": 0.359, + "hit@5": 0.629, + "hit@10": 0.732, + "ndcg_linear@1": 0.359, + "ndcg_linear@5": 0.5027089043672949, + "ndcg_linear@10": 0.5357119262717347, + "ndcg_exponential@1": 0.359, + "ndcg_exponential@5": 0.5027089043672949, + "ndcg_exponential@10": 0.5357119262717347, + "precision@1": 0.359, + "precision@5": 0.1258, + "precision@10": 0.07319999999999999, + "recall@1": 0.359, + "recall@5": 0.629, + "recall@10": 0.732, + "f1@1": 0.359, + "f1@5": 0.20966666666666672, + "f1@10": 0.13309090909090912, + "map@1": 0.359, + "map@5": 0.4606666666666666, + "map@10": 0.47410833333333335, + "mrr@1": 0.359, + "mrr@5": 0.4606666666666666, + "mrr@10": 0.47410833333333335, "num_pred": 1000, "num_data": 1000 }, "ImageNet-A": { "hit@1": 0.363, - "hit@5": 0.632, - "hit@10": 0.717, + "hit@5": 0.635, + "hit@10": 0.729, "ndcg_linear@1": 0.363, - "ndcg_linear@5": 0.5030434388907347, - "ndcg_linear@10": 0.5307926856684906, + "ndcg_linear@5": 0.5057111997434085, + "ndcg_linear@10": 0.5363863845507809, "ndcg_exponential@1": 0.363, - "ndcg_exponential@5": 0.5030434388907347, - "ndcg_exponential@10": 0.5307926856684906, + "ndcg_exponential@5": 0.5057111997434085, + "ndcg_exponential@10": 0.5363863845507809, "precision@1": 0.363, - "precision@5": 0.12639999999999998, - "precision@10": 0.0717, + "precision@5": 0.127, + "precision@10": 0.0729, "recall@1": 0.363, - "recall@5": 0.632, - "recall@10": 0.717, + "recall@5": 0.635, + "recall@10": 0.729, "f1@1": 0.363, - "f1@5": 0.21066666666666675, - "f1@10": 0.13036363636363638, + "f1@5": 0.21166666666666675, + "f1@10": 0.13254545454545455, "map@1": 0.363, - "map@5": 0.46033333333333326, - "map@10": 0.471936111111111, + "map@5": 0.4628166666666666, + "map@10": 0.47564246031746027, "mrr@1": 0.363, - "mrr@5": 0.46033333333333326, - "mrr@10": 0.471936111111111, + "mrr@5": 0.4628166666666666, + "mrr@10": 0.47564246031746027, "num_pred": 1000, "num_data": 1000 }, "ImageNet-R": { "hit@1": 0.77, - "hit@5": 0.926, - "hit@10": 0.95, + "hit@5": 0.918, + "hit@10": 0.953, "ndcg_linear@1": 0.77, - "ndcg_linear@5": 0.8544616508193265, - "ndcg_linear@10": 0.8620257015631435, + "ndcg_linear@5": 0.8505363430912923, + "ndcg_linear@10": 0.8618736380822317, "ndcg_exponential@1": 0.77, - "ndcg_exponential@5": 0.8544616508193265, - "ndcg_exponential@10": 0.8620257015631435, + "ndcg_exponential@5": 0.8505363430912923, + "ndcg_exponential@10": 0.8618736380822317, "precision@1": 0.77, - "precision@5": 0.18520000000000006, - "precision@10": 0.09500000000000001, + "precision@5": 0.1836, + "precision@10": 0.09530000000000002, "recall@1": 0.77, - "recall@5": 0.926, - "recall@10": 0.95, + "recall@5": 0.918, + "recall@10": 0.953, "f1@1": 0.77, - "f1@5": 0.3086666666666667, - "f1@10": 0.1727272727272727, + "f1@5": 0.306, + "f1@10": 0.17327272727272724, "map@1": 0.77, - "map@5": 0.8305166666666667, - "map@10": 0.8335222222222223, + "map@5": 0.8279000000000001, + "map@10": 0.8325904761904762, "mrr@1": 0.77, - "mrr@5": 0.8305166666666667, - "mrr@10": 0.8335222222222223, + "mrr@5": 0.8279000000000001, + "mrr@10": 0.8325904761904762, "num_pred": 1000, "num_data": 1000 }, "ObjectNet": { - "hit@1": 0.594, - "hit@5": 0.797, - "hit@10": 0.866, - "ndcg_linear@1": 0.594, - "ndcg_linear@5": 0.7039724929773868, - "ndcg_linear@10": 0.7264585203163247, - "ndcg_exponential@1": 0.594, - "ndcg_exponential@5": 0.7039724929773868, - "ndcg_exponential@10": 0.7264585203163247, - "precision@1": 0.594, - "precision@5": 0.15940000000000001, - "precision@10": 0.08660000000000001, - "recall@1": 0.594, - "recall@5": 0.797, - "recall@10": 0.866, - "f1@1": 0.594, - "f1@5": 0.2656666666666667, - "f1@10": 0.15745454545454543, - "map@1": 0.594, - "map@5": 0.67285, - "map@10": 0.6822277777777778, - "mrr@1": 0.594, - "mrr@5": 0.67285, - "mrr@10": 0.6822277777777778, + "hit@1": 0.592, + "hit@5": 0.804, + "hit@10": 0.864, + "ndcg_linear@1": 0.592, + "ndcg_linear@5": 0.7077711835836321, + "ndcg_linear@10": 0.7272631408573631, + "ndcg_exponential@1": 0.592, + "ndcg_exponential@5": 0.7077711835836321, + "ndcg_exponential@10": 0.7272631408573631, + "precision@1": 0.592, + "precision@5": 0.1608, + "precision@10": 0.0864, + "recall@1": 0.592, + "recall@5": 0.804, + "recall@10": 0.864, + "f1@1": 0.592, + "f1@5": 0.26800000000000007, + "f1@10": 0.1570909090909091, + "map@1": 0.592, + "map@5": 0.6755, + "map@10": 0.6835956349206349, + "mrr@1": 0.592, + "mrr@5": 0.6755, + "mrr@10": 0.6835956349206349, "num_pred": 1000, "num_data": 1000 }, "Country211": { - "hit@1": 0.217, - "hit@5": 0.454, - "hit@10": 0.557, - "ndcg_linear@1": 0.217, - "ndcg_linear@5": 0.3418498205027662, - "ndcg_linear@10": 0.37474993648572913, - "ndcg_exponential@1": 0.217, - "ndcg_exponential@5": 0.3418498205027662, - "ndcg_exponential@10": 0.37474993648572913, - "precision@1": 0.217, - "precision@5": 0.09079999999999999, - "precision@10": 0.05569999999999999, - "recall@1": 0.217, - "recall@5": 0.454, - "recall@10": 0.557, - "f1@1": 0.217, - "f1@5": 0.15133333333333338, - "f1@10": 0.10127272727272728, - "map@1": 0.217, - "map@5": 0.3046333333333333, - "map@10": 0.31797579365079365, - "mrr@1": 0.217, - "mrr@5": 0.3046333333333333, - "mrr@10": 0.31797579365079365, + "hit@1": 0.206, + "hit@5": 0.448, + "hit@10": 0.542, + "ndcg_linear@1": 0.206, + "ndcg_linear@5": 0.33388541174887365, + "ndcg_linear@10": 0.3639241045927329, + "ndcg_exponential@1": 0.206, + "ndcg_exponential@5": 0.33388541174887365, + "ndcg_exponential@10": 0.3639241045927329, + "precision@1": 0.206, + "precision@5": 0.0896, + "precision@10": 0.054200000000000005, + "recall@1": 0.206, + "recall@5": 0.448, + "recall@10": 0.542, + "f1@1": 0.206, + "f1@5": 0.14933333333333337, + "f1@10": 0.09854545454545456, + "map@1": 0.206, + "map@5": 0.29605000000000004, + "map@10": 0.30824126984126987, + "mrr@1": 0.206, + "mrr@5": 0.29605000000000004, + "mrr@10": 0.30824126984126987, "num_pred": 1000, "num_data": 1000 }, "OK-VQA": { - "hit@1": 0.399, - "hit@5": 0.698, - "hit@10": 0.802, - "ndcg_linear@1": 0.399, - "ndcg_linear@5": 0.5564084846581007, - "ndcg_linear@10": 0.5902611648290425, - "ndcg_exponential@1": 0.399, - "ndcg_exponential@5": 0.5564084846581007, - "ndcg_exponential@10": 0.5902611648290425, - "precision@1": 0.399, - "precision@5": 0.13959999999999997, - "precision@10": 0.0802, - "recall@1": 0.399, - "recall@5": 0.698, - "recall@10": 0.802, - "f1@1": 0.399, - "f1@5": 0.23266666666666674, - "f1@10": 0.14581818181818185, - "map@1": 0.399, - "map@5": 0.5095, - "map@10": 0.523604365079365, - "mrr@1": 0.399, - "mrr@5": 0.5095, - "mrr@10": 0.523604365079365, + "hit@1": 0.377, + "hit@5": 0.666, + "hit@10": 0.779, + "ndcg_linear@1": 0.377, + "ndcg_linear@5": 0.5320370935862011, + "ndcg_linear@10": 0.5686392586235357, + "ndcg_exponential@1": 0.377, + "ndcg_exponential@5": 0.5320370935862011, + "ndcg_exponential@10": 0.5686392586235357, + "precision@1": 0.377, + "precision@5": 0.13319999999999999, + "precision@10": 0.07790000000000001, + "recall@1": 0.377, + "recall@5": 0.666, + "recall@10": 0.779, + "f1@1": 0.377, + "f1@5": 0.22200000000000006, + "f1@10": 0.14163636363636364, + "map@1": 0.377, + "map@5": 0.48735, + "map@10": 0.502484126984127, + "mrr@1": 0.377, + "mrr@5": 0.48735, + "mrr@10": 0.502484126984127, "num_pred": 1000, "num_data": 1000 }, "A-OKVQA": { - "hit@1": 0.341, - "hit@5": 0.657, - "hit@10": 0.778, - "ndcg_linear@1": 0.341, - "ndcg_linear@5": 0.5094458840557816, - "ndcg_linear@10": 0.5493145190145722, - "ndcg_exponential@1": 0.341, - "ndcg_exponential@5": 0.5094458840557816, - "ndcg_exponential@10": 0.5493145190145722, - "precision@1": 0.341, - "precision@5": 0.1314, - "precision@10": 0.07780000000000001, - "recall@1": 0.341, - "recall@5": 0.657, - "recall@10": 0.778, - "f1@1": 0.341, - "f1@5": 0.21900000000000006, - "f1@10": 0.14145454545454547, - "map@1": 0.341, - "map@5": 0.4603333333333333, - "map@10": 0.4772162698412698, - "mrr@1": 0.341, - "mrr@5": 0.4603333333333333, - "mrr@10": 0.4772162698412698, + "hit@1": 0.312, + "hit@5": 0.639, + "hit@10": 0.755, + "ndcg_linear@1": 0.312, + "ndcg_linear@5": 0.48745886608512695, + "ndcg_linear@10": 0.5252343937112955, + "ndcg_exponential@1": 0.312, + "ndcg_exponential@5": 0.48745886608512695, + "ndcg_exponential@10": 0.5252343937112955, + "precision@1": 0.312, + "precision@5": 0.12779999999999997, + "precision@10": 0.0755, + "recall@1": 0.312, + "recall@5": 0.639, + "recall@10": 0.755, + "f1@1": 0.312, + "f1@5": 0.21300000000000005, + "f1@10": 0.13727272727272727, + "map@1": 0.312, + "map@5": 0.4369166666666667, + "map@10": 0.45265515873015877, + "mrr@1": 0.312, + "mrr@5": 0.4369166666666667, + "mrr@10": 0.45265515873015877, "num_pred": 1000, "num_data": 1000 }, "DocVQA": { - "hit@1": 0.371, - "hit@5": 0.554, - "hit@10": 0.624, - "ndcg_linear@1": 0.371, - "ndcg_linear@5": 0.46897587135139057, - "ndcg_linear@10": 0.49181251916964985, - "ndcg_exponential@1": 0.371, - "ndcg_exponential@5": 0.46897587135139057, - "ndcg_exponential@10": 0.49181251916964985, - "precision@1": 0.371, - "precision@5": 0.1108, - "precision@10": 0.06239999999999999, - "recall@1": 0.371, - "recall@5": 0.554, - "recall@10": 0.624, - "f1@1": 0.371, - "f1@5": 0.1846666666666667, - "f1@10": 0.11345454545454549, - "map@1": 0.371, - "map@5": 0.4406333333333333, - "map@10": 0.45016785714285706, - "mrr@1": 0.371, - "mrr@5": 0.4406333333333333, - "mrr@10": 0.45016785714285706, + "hit@1": 0.379, + "hit@5": 0.562, + "hit@10": 0.63, + "ndcg_linear@1": 0.379, + "ndcg_linear@5": 0.47680594517807445, + "ndcg_linear@10": 0.4989533825702415, + "ndcg_exponential@1": 0.379, + "ndcg_exponential@5": 0.47680594517807445, + "ndcg_exponential@10": 0.4989533825702415, + "precision@1": 0.379, + "precision@5": 0.11239999999999999, + "precision@10": 0.063, + "recall@1": 0.379, + "recall@5": 0.562, + "recall@10": 0.63, + "f1@1": 0.379, + "f1@5": 0.18733333333333338, + "f1@10": 0.11454545454545456, + "map@1": 0.379, + "map@5": 0.44839999999999997, + "map@10": 0.457634126984127, + "mrr@1": 0.379, + "mrr@5": 0.44839999999999997, + "mrr@10": 0.457634126984127, "num_pred": 1000, "num_data": 1000 }, "InfographicsVQA": { - "hit@1": 0.237, - "hit@5": 0.393, - "hit@10": 0.467, - "ndcg_linear@1": 0.237, - "ndcg_linear@5": 0.3197525706960229, - "ndcg_linear@10": 0.3435396532981073, - "ndcg_exponential@1": 0.237, - "ndcg_exponential@5": 0.3197525706960229, - "ndcg_exponential@10": 0.3435396532981073, - "precision@1": 0.237, - "precision@5": 0.07859999999999999, - "precision@10": 0.0467, - "recall@1": 0.237, - "recall@5": 0.393, - "recall@10": 0.467, - "f1@1": 0.237, - "f1@5": 0.13100000000000003, - "f1@10": 0.08490909090909092, - "map@1": 0.237, - "map@5": 0.2954333333333333, - "map@10": 0.30516269841269844, - "mrr@1": 0.237, - "mrr@5": 0.2954333333333333, - "mrr@10": 0.30516269841269844, + "hit@1": 0.232, + "hit@5": 0.385, + "hit@10": 0.456, + "ndcg_linear@1": 0.232, + "ndcg_linear@5": 0.31320928844412926, + "ndcg_linear@10": 0.33645722801967326, + "ndcg_exponential@1": 0.232, + "ndcg_exponential@5": 0.31320928844412926, + "ndcg_exponential@10": 0.33645722801967326, + "precision@1": 0.232, + "precision@5": 0.07699999999999999, + "precision@10": 0.0456, + "recall@1": 0.232, + "recall@5": 0.385, + "recall@10": 0.456, + "f1@1": 0.232, + "f1@5": 0.12833333333333338, + "f1@10": 0.08290909090909092, + "map@1": 0.232, + "map@5": 0.28936666666666666, + "map@10": 0.29912619047619043, + "mrr@1": 0.232, + "mrr@5": 0.28936666666666666, + "mrr@10": 0.29912619047619043, "num_pred": 1000, "num_data": 1000 }, "ChartQA": { - "hit@1": 0.15, - "hit@5": 0.212, - "hit@10": 0.243, - "ndcg_linear@1": 0.15, - "ndcg_linear@5": 0.18326787433410913, - "ndcg_linear@10": 0.19342783194595115, - "ndcg_exponential@1": 0.15, - "ndcg_exponential@5": 0.18326787433410913, - "ndcg_exponential@10": 0.19342783194595115, - "precision@1": 0.15, - "precision@5": 0.04239999999999999, - "precision@10": 0.024300000000000002, - "recall@1": 0.15, - "recall@5": 0.212, - "recall@10": 0.243, - "f1@1": 0.15, - "f1@5": 0.07066666666666668, - "f1@10": 0.04418181818181819, - "map@1": 0.15, - "map@5": 0.17368333333333333, - "map@10": 0.17795158730158733, - "mrr@1": 0.15, - "mrr@5": 0.17368333333333333, - "mrr@10": 0.17795158730158733, + "hit@1": 0.145, + "hit@5": 0.199, + "hit@10": 0.233, + "ndcg_linear@1": 0.145, + "ndcg_linear@5": 0.17410909658920748, + "ndcg_linear@10": 0.18514651069066765, + "ndcg_exponential@1": 0.145, + "ndcg_exponential@5": 0.17410909658920748, + "ndcg_exponential@10": 0.18514651069066765, + "precision@1": 0.145, + "precision@5": 0.039799999999999995, + "precision@10": 0.0233, + "recall@1": 0.145, + "recall@5": 0.199, + "recall@10": 0.233, + "f1@1": 0.145, + "f1@5": 0.06633333333333334, + "f1@10": 0.042363636363636374, + "map@1": 0.145, + "map@5": 0.16578333333333334, + "map@10": 0.17035833333333336, + "mrr@1": 0.145, + "mrr@5": 0.16578333333333334, + "mrr@10": 0.17035833333333336, "num_pred": 1000, "num_data": 1000 }, "Visual7W": { - "hit@1": 0.246, - "hit@5": 0.529, - "hit@10": 0.646, - "ndcg_linear@1": 0.246, - "ndcg_linear@5": 0.39569651608588563, - "ndcg_linear@10": 0.4329434083919529, - "ndcg_exponential@1": 0.246, - "ndcg_exponential@5": 0.39569651608588563, - "ndcg_exponential@10": 0.4329434083919529, - "precision@1": 0.246, - "precision@5": 0.10579999999999999, - "precision@10": 0.06459999999999999, - "recall@1": 0.246, - "recall@5": 0.529, - "recall@10": 0.646, - "f1@1": 0.246, - "f1@5": 0.17633333333333337, - "f1@10": 0.11745454545454546, - "map@1": 0.246, - "map@5": 0.3514333333333333, - "map@10": 0.36646388888888887, - "mrr@1": 0.246, - "mrr@5": 0.3514333333333333, - "mrr@10": 0.36646388888888887, + "hit@1": 0.244, + "hit@5": 0.525, + "hit@10": 0.655, + "ndcg_linear@1": 0.244, + "ndcg_linear@5": 0.3926774682227273, + "ndcg_linear@10": 0.4344801063970569, + "ndcg_exponential@1": 0.244, + "ndcg_exponential@5": 0.3926774682227273, + "ndcg_exponential@10": 0.4344801063970569, + "precision@1": 0.244, + "precision@5": 0.105, + "precision@10": 0.0655, + "recall@1": 0.244, + "recall@5": 0.525, + "recall@10": 0.655, + "f1@1": 0.244, + "f1@5": 0.17500000000000004, + "f1@10": 0.11909090909090911, + "map@1": 0.244, + "map@5": 0.3486666666666666, + "map@10": 0.36577936507936504, + "mrr@1": 0.244, + "mrr@5": 0.3486666666666666, + "mrr@10": 0.36577936507936504, "num_pred": 1000, "num_data": 1000 }, "ScienceQA": { - "hit@1": 0.313, - "hit@5": 0.657, - "hit@10": 0.747, - "ndcg_linear@1": 0.313, - "ndcg_linear@5": 0.4996758463183123, - "ndcg_linear@10": 0.52897999298943, - "ndcg_exponential@1": 0.313, - "ndcg_exponential@5": 0.4996758463183123, - "ndcg_exponential@10": 0.52897999298943, - "precision@1": 0.313, - "precision@5": 0.1314, - "precision@10": 0.0747, - "recall@1": 0.313, - "recall@5": 0.657, - "recall@10": 0.747, - "f1@1": 0.313, - "f1@5": 0.21900000000000006, - "f1@10": 0.1358181818181818, - "map@1": 0.313, - "map@5": 0.44695, - "map@10": 0.4591571428571428, - "mrr@1": 0.313, - "mrr@5": 0.44695, - "mrr@10": 0.4591571428571428, + "hit@1": 0.317, + "hit@5": 0.664, + "hit@10": 0.748, + "ndcg_linear@1": 0.317, + "ndcg_linear@5": 0.5045068462877816, + "ndcg_linear@10": 0.5315292037144778, + "ndcg_exponential@1": 0.317, + "ndcg_exponential@5": 0.5045068462877816, + "ndcg_exponential@10": 0.5315292037144778, + "precision@1": 0.317, + "precision@5": 0.13279999999999997, + "precision@10": 0.07479999999999999, + "recall@1": 0.317, + "recall@5": 0.664, + "recall@10": 0.748, + "f1@1": 0.317, + "f1@5": 0.2213333333333334, + "f1@10": 0.136, + "map@1": 0.317, + "map@5": 0.45108333333333334, + "map@10": 0.46214880952380955, + "mrr@1": 0.317, + "mrr@5": 0.45108333333333334, + "mrr@10": 0.46214880952380955, "num_pred": 1000, "num_data": 1000 }, "VizWiz": { "hit@1": 0.32, - "hit@5": 0.53, - "hit@10": 0.608, + "hit@5": 0.527, + "hit@10": 0.601, "ndcg_linear@1": 0.32, - "ndcg_linear@5": 0.43200857279831045, - "ndcg_linear@10": 0.4574650966057038, + "ndcg_linear@5": 0.4317124402715414, + "ndcg_linear@10": 0.4558187261527039, "ndcg_exponential@1": 0.32, - "ndcg_exponential@5": 0.43200857279831045, - "ndcg_exponential@10": 0.4574650966057038, + "ndcg_exponential@5": 0.4317124402715414, + "ndcg_exponential@10": 0.4558187261527039, "precision@1": 0.32, - "precision@5": 0.106, - "precision@10": 0.0608, + "precision@5": 0.1054, + "precision@10": 0.060099999999999994, "recall@1": 0.32, - "recall@5": 0.53, - "recall@10": 0.608, + "recall@5": 0.527, + "recall@10": 0.601, "f1@1": 0.32, - "f1@5": 0.17666666666666675, - "f1@10": 0.11054545454545456, + "f1@5": 0.17566666666666672, + "f1@10": 0.1092727272727273, "map@1": 0.32, - "map@5": 0.39936666666666665, - "map@10": 0.4100103174603174, + "map@5": 0.3999, + "map@10": 0.4099555555555555, "mrr@1": 0.32, - "mrr@5": 0.39936666666666665, - "mrr@10": 0.4100103174603174, + "mrr@5": 0.3999, + "mrr@10": 0.4099555555555555, "num_pred": 1000, "num_data": 1000 }, "GQA": { - "hit@1": 0.574, - "hit@5": 0.878, - "hit@10": 0.938, - "ndcg_linear@1": 0.574, - "ndcg_linear@5": 0.741998771313293, - "ndcg_linear@10": 0.7614732426835671, - "ndcg_exponential@1": 0.574, - "ndcg_exponential@5": 0.741998771313293, - "ndcg_exponential@10": 0.7614732426835671, - "precision@1": 0.574, - "precision@5": 0.17560000000000003, - "precision@10": 0.09380000000000002, - "recall@1": 0.574, - "recall@5": 0.878, - "recall@10": 0.938, - "f1@1": 0.574, - "f1@5": 0.29266666666666674, - "f1@10": 0.17054545454545453, - "map@1": 0.574, - "map@5": 0.6962333333333334, - "map@10": 0.7043083333333333, - "mrr@1": 0.574, - "mrr@5": 0.6962333333333334, - "mrr@10": 0.7043083333333333, + "hit@1": 0.55, + "hit@5": 0.867, + "hit@10": 0.929, + "ndcg_linear@1": 0.55, + "ndcg_linear@5": 0.7222760376062278, + "ndcg_linear@10": 0.7425036656610403, + "ndcg_exponential@1": 0.55, + "ndcg_exponential@5": 0.7222760376062278, + "ndcg_exponential@10": 0.7425036656610403, + "precision@1": 0.55, + "precision@5": 0.1734, + "precision@10": 0.09290000000000001, + "recall@1": 0.55, + "recall@5": 0.867, + "recall@10": 0.929, + "f1@1": 0.55, + "f1@5": 0.289, + "f1@10": 0.1689090909090909, + "map@1": 0.55, + "map@5": 0.67385, + "map@10": 0.6822999999999999, + "mrr@1": 0.55, + "mrr@5": 0.67385, + "mrr@10": 0.6822999999999999, "num_pred": 1000, "num_data": 1000 }, "TextVQA": { - "hit@1": 0.461, - "hit@5": 0.663, - "hit@10": 0.745, - "ndcg_linear@1": 0.461, - "ndcg_linear@5": 0.57111707631472, - "ndcg_linear@10": 0.5979045563649381, - "ndcg_exponential@1": 0.461, - "ndcg_exponential@5": 0.57111707631472, - "ndcg_exponential@10": 0.5979045563649381, - "precision@1": 0.461, - "precision@5": 0.13259999999999997, - "precision@10": 0.0745, - "recall@1": 0.461, - "recall@5": 0.663, - "recall@10": 0.745, - "f1@1": 0.461, - "f1@5": 0.22100000000000006, - "f1@10": 0.1354545454545455, - "map@1": 0.461, - "map@5": 0.5403166666666666, - "map@10": 0.5515261904761906, - "mrr@1": 0.461, - "mrr@5": 0.5403166666666666, - "mrr@10": 0.5515261904761906, + "hit@1": 0.455, + "hit@5": 0.673, + "hit@10": 0.744, + "ndcg_linear@1": 0.455, + "ndcg_linear@5": 0.5726928770339335, + "ndcg_linear@10": 0.595679695627728, + "ndcg_exponential@1": 0.455, + "ndcg_exponential@5": 0.5726928770339335, + "ndcg_exponential@10": 0.595679695627728, + "precision@1": 0.455, + "precision@5": 0.13459999999999997, + "precision@10": 0.07440000000000001, + "recall@1": 0.455, + "recall@5": 0.673, + "recall@10": 0.744, + "f1@1": 0.455, + "f1@5": 0.22433333333333338, + "f1@10": 0.13527272727272727, + "map@1": 0.455, + "map@5": 0.5391666666666667, + "map@10": 0.5486714285714286, + "mrr@1": 0.455, + "mrr@5": 0.5391666666666667, + "mrr@10": 0.5486714285714286, "num_pred": 1000, "num_data": 1000 }, "VisDial": { - "hit@1": 0.625, - "hit@5": 0.867, - "hit@10": 0.932, - "ndcg_linear@1": 0.625, - "ndcg_linear@5": 0.755047080643239, - "ndcg_linear@10": 0.7763706281998177, - "ndcg_exponential@1": 0.625, - "ndcg_exponential@5": 0.755047080643239, - "ndcg_exponential@10": 0.7763706281998177, - "precision@1": 0.625, - "precision@5": 0.17340000000000003, - "precision@10": 0.09320000000000002, - "recall@1": 0.625, - "recall@5": 0.867, - "recall@10": 0.932, - "f1@1": 0.625, - "f1@5": 0.28900000000000003, - "f1@10": 0.16945454545454544, - "map@1": 0.625, - "map@5": 0.7176666666666667, - "map@10": 0.7266424603174603, - "mrr@1": 0.625, - "mrr@5": 0.7176666666666667, - "mrr@10": 0.7266424603174603, + "hit@1": 0.63, + "hit@5": 0.875, + "hit@10": 0.936, + "ndcg_linear@1": 0.63, + "ndcg_linear@5": 0.7624280237966272, + "ndcg_linear@10": 0.7825534206314273, + "ndcg_exponential@1": 0.63, + "ndcg_exponential@5": 0.7624280237966272, + "ndcg_exponential@10": 0.7825534206314273, + "precision@1": 0.63, + "precision@5": 0.17500000000000002, + "precision@10": 0.09360000000000002, + "recall@1": 0.63, + "recall@5": 0.875, + "recall@10": 0.936, + "f1@1": 0.63, + "f1@5": 0.29166666666666674, + "f1@10": 0.17018181818181818, + "map@1": 0.63, + "map@5": 0.7247666666666667, + "map@10": 0.7333051587301587, + "mrr@1": 0.63, + "mrr@5": 0.7247666666666667, + "mrr@10": 0.7333051587301587, "num_pred": 1000, "num_data": 1000 }, "CIRR": { - "hit@1": 0.447, - "hit@5": 0.806, - "hit@10": 0.886, - "ndcg_linear@1": 0.447, - "ndcg_linear@5": 0.6402518032174924, - "ndcg_linear@10": 0.6664724583675284, - "ndcg_exponential@1": 0.447, - "ndcg_exponential@5": 0.6402518032174924, - "ndcg_exponential@10": 0.6664724583675284, - "precision@1": 0.447, - "precision@5": 0.16119999999999998, - "precision@10": 0.08860000000000003, - "recall@1": 0.447, - "recall@5": 0.806, - "recall@10": 0.886, - "f1@1": 0.447, - "f1@5": 0.2686666666666667, - "f1@10": 0.1610909090909091, - "map@1": 0.447, - "map@5": 0.5848833333333333, - "map@10": 0.5959150793650794, - "mrr@1": 0.447, - "mrr@5": 0.5848833333333333, - "mrr@10": 0.5959150793650794, + "hit@1": 0.466, + "hit@5": 0.82, + "hit@10": 0.902, + "ndcg_linear@1": 0.466, + "ndcg_linear@5": 0.6586683121676306, + "ndcg_linear@10": 0.6855807400768018, + "ndcg_exponential@1": 0.466, + "ndcg_exponential@5": 0.6586683121676306, + "ndcg_exponential@10": 0.6855807400768018, + "precision@1": 0.466, + "precision@5": 0.164, + "precision@10": 0.09020000000000002, + "recall@1": 0.466, + "recall@5": 0.82, + "recall@10": 0.902, + "f1@1": 0.466, + "f1@5": 0.2733333333333334, + "f1@10": 0.164, + "map@1": 0.466, + "map@5": 0.6045666666666666, + "map@10": 0.6158988095238095, + "mrr@1": 0.466, + "mrr@5": 0.6045666666666666, + "mrr@10": 0.6158988095238095, "num_pred": 1000, "num_data": 1000 }, "VisualNews_t2i": { - "hit@1": 0.701, - "hit@5": 0.876, - "hit@10": 0.915, - "ndcg_linear@1": 0.701, - "ndcg_linear@5": 0.795372764139792, - "ndcg_linear@10": 0.8079777293038085, - "ndcg_exponential@1": 0.701, - "ndcg_exponential@5": 0.795372764139792, - "ndcg_exponential@10": 0.8079777293038085, - "precision@1": 0.701, - "precision@5": 0.1752, - "precision@10": 0.09150000000000001, - "recall@1": 0.701, - "recall@5": 0.876, - "recall@10": 0.915, - "f1@1": 0.701, - "f1@5": 0.29200000000000004, - "f1@10": 0.16636363636363638, - "map@1": 0.701, - "map@5": 0.7683666666666666, - "map@10": 0.7735666666666666, - "mrr@1": 0.701, - "mrr@5": 0.7683666666666666, - "mrr@10": 0.7735666666666666, + "hit@1": 0.699, + "hit@5": 0.866, + "hit@10": 0.91, + "ndcg_linear@1": 0.699, + "ndcg_linear@5": 0.7904196606694185, + "ndcg_linear@10": 0.8049141802093887, + "ndcg_exponential@1": 0.699, + "ndcg_exponential@5": 0.7904196606694185, + "ndcg_exponential@10": 0.8049141802093887, + "precision@1": 0.699, + "precision@5": 0.17320000000000002, + "precision@10": 0.091, + "recall@1": 0.699, + "recall@5": 0.866, + "recall@10": 0.91, + "f1@1": 0.699, + "f1@5": 0.2886666666666667, + "f1@10": 0.16545454545454544, + "map@1": 0.699, + "map@5": 0.7650500000000001, + "map@10": 0.7711837301587302, + "mrr@1": 0.699, + "mrr@5": 0.7650500000000001, + "mrr@10": 0.7711837301587302, "num_pred": 1000, "num_data": 1000 }, "VisualNews_i2t": { - "hit@1": 0.742, - "hit@5": 0.905, - "hit@10": 0.94, - "ndcg_linear@1": 0.742, - "ndcg_linear@5": 0.8316797133752641, - "ndcg_linear@10": 0.8432770462154437, - "ndcg_exponential@1": 0.742, - "ndcg_exponential@5": 0.8316797133752641, - "ndcg_exponential@10": 0.8432770462154437, - "precision@1": 0.742, - "precision@5": 0.181, - "precision@10": 0.094, - "recall@1": 0.742, - "recall@5": 0.905, - "recall@10": 0.94, - "f1@1": 0.742, - "f1@5": 0.3016666666666667, - "f1@10": 0.17090909090909087, - "map@1": 0.742, - "map@5": 0.807, - "map@10": 0.8119500000000001, - "mrr@1": 0.742, - "mrr@5": 0.807, - "mrr@10": 0.8119500000000001, + "hit@1": 0.755, + "hit@5": 0.912, + "hit@10": 0.946, + "ndcg_linear@1": 0.755, + "ndcg_linear@5": 0.8401874037059122, + "ndcg_linear@10": 0.8512584417506328, + "ndcg_exponential@1": 0.755, + "ndcg_exponential@5": 0.8401874037059122, + "ndcg_exponential@10": 0.8512584417506328, + "precision@1": 0.755, + "precision@5": 0.18240000000000003, + "precision@10": 0.0946, + "recall@1": 0.755, + "recall@5": 0.912, + "recall@10": 0.946, + "f1@1": 0.755, + "f1@5": 0.304, + "f1@10": 0.172, + "map@1": 0.755, + "map@5": 0.8161333333333334, + "map@10": 0.8207472222222223, + "mrr@1": 0.755, + "mrr@5": 0.8161333333333334, + "mrr@10": 0.8207472222222223, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_t2i": { - "hit@1": 0.657, - "hit@5": 0.891, - "hit@10": 0.939, - "ndcg_linear@1": 0.657, - "ndcg_linear@5": 0.7847776876605524, - "ndcg_linear@10": 0.8004002047017017, - "ndcg_exponential@1": 0.657, - "ndcg_exponential@5": 0.7847776876605524, - "ndcg_exponential@10": 0.8004002047017017, - "precision@1": 0.657, - "precision@5": 0.1782, - "precision@10": 0.09390000000000001, - "recall@1": 0.657, - "recall@5": 0.891, - "recall@10": 0.939, - "f1@1": 0.657, - "f1@5": 0.297, - "f1@10": 0.17072727272727273, - "map@1": 0.657, - "map@5": 0.7491333333333333, - "map@10": 0.7556400793650793, - "mrr@1": 0.657, - "mrr@5": 0.7491333333333333, - "mrr@10": 0.7556400793650793, + "hit@1": 0.68, + "hit@5": 0.894, + "hit@10": 0.949, + "ndcg_linear@1": 0.68, + "ndcg_linear@5": 0.7964831005819942, + "ndcg_linear@10": 0.8145484074493904, + "ndcg_exponential@1": 0.68, + "ndcg_exponential@5": 0.7964831005819942, + "ndcg_exponential@10": 0.8145484074493904, + "precision@1": 0.68, + "precision@5": 0.17880000000000001, + "precision@10": 0.09490000000000001, + "recall@1": 0.68, + "recall@5": 0.894, + "recall@10": 0.949, + "f1@1": 0.68, + "f1@5": 0.298, + "f1@10": 0.17254545454545453, + "map@1": 0.68, + "map@5": 0.7637166666666667, + "map@10": 0.7713337301587302, + "mrr@1": 0.68, + "mrr@5": 0.7637166666666667, + "mrr@10": 0.7713337301587302, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_i2t": { - "hit@1": 0.711, - "hit@5": 0.93, - "hit@10": 0.972, - "ndcg_linear@1": 0.711, - "ndcg_linear@5": 0.8324157937205986, - "ndcg_linear@10": 0.8463743391451135, - "ndcg_exponential@1": 0.711, - "ndcg_exponential@5": 0.8324157937205986, - "ndcg_exponential@10": 0.8463743391451135, - "precision@1": 0.711, - "precision@5": 0.186, - "precision@10": 0.09720000000000002, - "recall@1": 0.711, - "recall@5": 0.93, - "recall@10": 0.972, - "f1@1": 0.711, - "f1@5": 0.31, - "f1@10": 0.17672727272727273, - "map@1": 0.711, - "map@5": 0.7995, - "map@10": 0.805479761904762, - "mrr@1": 0.711, - "mrr@5": 0.7995, - "mrr@10": 0.805479761904762, + "hit@1": 0.714, + "hit@5": 0.934, + "hit@10": 0.973, + "ndcg_linear@1": 0.714, + "ndcg_linear@5": 0.8351941698110572, + "ndcg_linear@10": 0.8482173219109107, + "ndcg_exponential@1": 0.714, + "ndcg_exponential@5": 0.8351941698110572, + "ndcg_exponential@10": 0.8482173219109107, + "precision@1": 0.714, + "precision@5": 0.18680000000000002, + "precision@10": 0.09730000000000003, + "recall@1": 0.714, + "recall@5": 0.934, + "recall@10": 0.973, + "f1@1": 0.714, + "f1@5": 0.3113333333333333, + "f1@10": 0.17690909090909088, + "map@1": 0.714, + "map@5": 0.8019499999999999, + "map@10": 0.8075646825396825, + "mrr@1": 0.714, + "mrr@5": 0.8019499999999999, + "mrr@10": 0.8075646825396825, "num_pred": 1000, "num_data": 1000 }, "NIGHTS": { - "hit@1": 0.644, - "hit@5": 0.974, - "hit@10": 0.995, - "ndcg_linear@1": 0.644, - "ndcg_linear@5": 0.8326028416077181, - "ndcg_linear@10": 0.8394925007397642, - "ndcg_exponential@1": 0.644, - "ndcg_exponential@5": 0.8326028416077181, - "ndcg_exponential@10": 0.8394925007397642, - "precision@1": 0.644, - "precision@5": 0.1948, - "precision@10": 0.09950000000000003, - "recall@1": 0.644, - "recall@5": 0.974, - "recall@10": 0.995, - "f1@1": 0.644, - "f1@5": 0.32466666666666666, - "f1@10": 0.18090909090909088, - "map@1": 0.644, - "map@5": 0.7843333333333333, - "map@10": 0.7872345238095239, - "mrr@1": 0.644, - "mrr@5": 0.7843333333333333, - "mrr@10": 0.7872345238095239, + "hit@1": 0.658, + "hit@5": 0.975, + "hit@10": 0.994, + "ndcg_linear@1": 0.658, + "ndcg_linear@5": 0.8384801436837399, + "ndcg_linear@10": 0.8446817414958787, + "ndcg_exponential@1": 0.658, + "ndcg_exponential@5": 0.8384801436837399, + "ndcg_exponential@10": 0.8446817414958787, + "precision@1": 0.658, + "precision@5": 0.19500000000000003, + "precision@10": 0.0994, + "recall@1": 0.658, + "recall@5": 0.975, + "recall@10": 0.994, + "f1@1": 0.658, + "f1@5": 0.325, + "f1@10": 0.1807272727272727, + "map@1": 0.658, + "map@5": 0.79195, + "map@10": 0.7945464285714285, + "mrr@1": 0.658, + "mrr@5": 0.79195, + "mrr@10": 0.7945464285714285, "num_pred": 1000, "num_data": 1000 }, "WebQA": { "hit@1": 0.857, "hit@5": 0.981, - "hit@10": 0.993, + "hit@10": 0.989, "ndcg_linear@1": 0.857, - "ndcg_linear@5": 0.930534232045055, - "ndcg_linear@10": 0.9345661261820422, + "ndcg_linear@5": 0.9278836854546086, + "ndcg_linear@10": 0.9306289844770536, "ndcg_exponential@1": 0.857, - "ndcg_exponential@5": 0.930534232045055, - "ndcg_exponential@10": 0.9345661261820422, + "ndcg_exponential@5": 0.9278836854546086, + "ndcg_exponential@10": 0.9306289844770536, "precision@1": 0.857, - "precision@5": 0.1962, - "precision@10": 0.09930000000000001, + "precision@5": 0.19620000000000004, + "precision@10": 0.09890000000000002, "recall@1": 0.857, "recall@5": 0.981, - "recall@10": 0.993, + "recall@10": 0.989, "f1@1": 0.857, "f1@5": 0.327, - "f1@10": 0.18054545454545454, + "f1@10": 0.17981818181818182, "map@1": 0.857, - "map@5": 0.9130666666666666, - "map@10": 0.9148186507936507, + "map@5": 0.9097666666666666, + "map@10": 0.9109928571428572, "mrr@1": 0.857, - "mrr@5": 0.9130666666666666, - "mrr@10": 0.9148186507936507, + "mrr@5": 0.9097666666666666, + "mrr@10": 0.9109928571428572, "num_pred": 1000, "num_data": 1000 }, "FashionIQ": { - "hit@1": 0.334, - "hit@5": 0.617, - "hit@10": 0.721, - "ndcg_linear@1": 0.334, - "ndcg_linear@5": 0.4835749279594312, - "ndcg_linear@10": 0.5173457667677877, - "ndcg_exponential@1": 0.334, - "ndcg_exponential@5": 0.4835749279594312, - "ndcg_exponential@10": 0.5173457667677877, - "precision@1": 0.334, - "precision@5": 0.12339999999999998, - "precision@10": 0.07210000000000001, - "recall@1": 0.334, - "recall@5": 0.617, - "recall@10": 0.721, - "f1@1": 0.334, - "f1@5": 0.20566666666666675, - "f1@10": 0.13109090909090912, - "map@1": 0.334, - "map@5": 0.4392166666666667, - "map@10": 0.4532317460317461, - "mrr@1": 0.334, - "mrr@5": 0.4392166666666667, - "mrr@10": 0.4532317460317461, + "hit@1": 0.347, + "hit@5": 0.635, + "hit@10": 0.745, + "ndcg_linear@1": 0.347, + "ndcg_linear@5": 0.4995311303330878, + "ndcg_linear@10": 0.5347012669320448, + "ndcg_exponential@1": 0.347, + "ndcg_exponential@5": 0.4995311303330878, + "ndcg_exponential@10": 0.5347012669320448, + "precision@1": 0.347, + "precision@5": 0.12699999999999997, + "precision@10": 0.0745, + "recall@1": 0.347, + "recall@5": 0.635, + "recall@10": 0.745, + "f1@1": 0.347, + "f1@5": 0.21166666666666673, + "f1@10": 0.13545454545454547, + "map@1": 0.347, + "map@5": 0.4545166666666666, + "map@10": 0.46880595238095235, + "mrr@1": 0.347, + "mrr@5": 0.4545166666666666, + "mrr@10": 0.46880595238095235, "num_pred": 1000, "num_data": 1000 }, "Wiki-SS-NQ": { - "hit@1": 0.67, - "hit@5": 0.871, - "hit@10": 0.914, - "ndcg_linear@1": 0.67, - "ndcg_linear@5": 0.781114754814111, - "ndcg_linear@10": 0.7951533968998946, - "ndcg_exponential@1": 0.67, - "ndcg_exponential@5": 0.781114754814111, - "ndcg_exponential@10": 0.7951533968998946, - "precision@1": 0.67, - "precision@5": 0.17420000000000002, - "precision@10": 0.09140000000000001, - "recall@1": 0.67, - "recall@5": 0.871, - "recall@10": 0.914, - "f1@1": 0.67, - "f1@5": 0.2903333333333334, - "f1@10": 0.16618181818181818, - "map@1": 0.67, - "map@5": 0.7508166666666667, - "map@10": 0.7566873015873016, - "mrr@1": 0.67, - "mrr@5": 0.7508166666666667, - "mrr@10": 0.7566873015873016, + "hit@1": 0.665, + "hit@5": 0.853, + "hit@10": 0.893, + "ndcg_linear@1": 0.665, + "ndcg_linear@5": 0.7671583687857162, + "ndcg_linear@10": 0.7800078001746861, + "ndcg_exponential@1": 0.665, + "ndcg_exponential@5": 0.7671583687857162, + "ndcg_exponential@10": 0.7800078001746861, + "precision@1": 0.665, + "precision@5": 0.17060000000000003, + "precision@10": 0.08930000000000002, + "recall@1": 0.665, + "recall@5": 0.853, + "recall@10": 0.893, + "f1@1": 0.665, + "f1@5": 0.2843333333333334, + "f1@10": 0.16236363636363635, + "map@1": 0.665, + "map@5": 0.7383833333333333, + "map@10": 0.7436313492063491, + "mrr@1": 0.665, + "mrr@5": 0.7383833333333333, + "mrr@10": 0.7436313492063491, "num_pred": 1000, "num_data": 1000 }, "OVEN": { - "hit@1": 0.848, - "hit@5": 0.969, - "hit@10": 0.984, - "ndcg_linear@1": 0.848, - "ndcg_linear@5": 0.9154393285131402, - "ndcg_linear@10": 0.9203325138418795, - "ndcg_exponential@1": 0.848, - "ndcg_exponential@5": 0.9154393285131402, - "ndcg_exponential@10": 0.9203325138418795, - "precision@1": 0.848, - "precision@5": 0.19380000000000003, - "precision@10": 0.09840000000000002, - "recall@1": 0.848, - "recall@5": 0.969, - "recall@10": 0.984, - "f1@1": 0.848, - "f1@5": 0.323, - "f1@10": 0.17890909090909088, - "map@1": 0.848, - "map@5": 0.89735, - "map@10": 0.8993952380952381, - "mrr@1": 0.848, - "mrr@5": 0.89735, - "mrr@10": 0.8993952380952381, + "hit@1": 0.85, + "hit@5": 0.97, + "hit@10": 0.982, + "ndcg_linear@1": 0.85, + "ndcg_linear@5": 0.9167396168165871, + "ndcg_linear@10": 0.9206651054829958, + "ndcg_exponential@1": 0.85, + "ndcg_exponential@5": 0.9167396168165871, + "ndcg_exponential@10": 0.9206651054829958, + "precision@1": 0.85, + "precision@5": 0.19400000000000003, + "precision@10": 0.09820000000000002, + "recall@1": 0.85, + "recall@5": 0.97, + "recall@10": 0.982, + "f1@1": 0.85, + "f1@5": 0.3233333333333333, + "f1@10": 0.1785454545454545, + "map@1": 0.85, + "map@5": 0.89875, + "map@10": 0.9003984126984128, + "mrr@1": 0.85, + "mrr@5": 0.89875, + "mrr@10": 0.9003984126984128, "num_pred": 1000, "num_data": 1000 }, "EDIS": { - "hit@1": 0.787, - "hit@5": 0.961, - "hit@10": 0.987, - "ndcg_linear@1": 0.787, - "ndcg_linear@5": 0.8832722404570268, - "ndcg_linear@10": 0.8919335060407184, - "ndcg_exponential@1": 0.787, - "ndcg_exponential@5": 0.8832722404570268, - "ndcg_exponential@10": 0.8919335060407184, - "precision@1": 0.787, - "precision@5": 0.19220000000000004, - "precision@10": 0.09870000000000002, - "recall@1": 0.787, - "recall@5": 0.961, - "recall@10": 0.987, - "f1@1": 0.787, - "f1@5": 0.32033333333333336, - "f1@10": 0.17945454545454545, - "map@1": 0.787, - "map@5": 0.8571166666666666, - "map@10": 0.860843253968254, - "mrr@1": 0.787, - "mrr@5": 0.8571166666666666, - "mrr@10": 0.860843253968254, + "hit@1": 0.805, + "hit@5": 0.969, + "hit@10": 0.988, + "ndcg_linear@1": 0.805, + "ndcg_linear@5": 0.8949428120602638, + "ndcg_linear@10": 0.9011117104667082, + "ndcg_exponential@1": 0.805, + "ndcg_exponential@5": 0.8949428120602638, + "ndcg_exponential@10": 0.9011117104667082, + "precision@1": 0.805, + "precision@5": 0.19380000000000003, + "precision@10": 0.09880000000000001, + "recall@1": 0.805, + "recall@5": 0.969, + "recall@10": 0.988, + "f1@1": 0.805, + "f1@5": 0.323, + "f1@10": 0.17963636363636362, + "map@1": 0.805, + "map@5": 0.87005, + "map@10": 0.8726051587301588, + "mrr@1": 0.805, + "mrr@5": 0.87005, + "mrr@10": 0.8726051587301588, "num_pred": 1000, "num_data": 1000 }, "MSCOCO": { - "hit@1": 0.36, - "hit@5": 0.552, - "hit@10": 0.65, - "ndcg_linear@1": 0.36, - "ndcg_linear@5": 0.46270632083846136, - "ndcg_linear@10": 0.49426183883135943, - "ndcg_exponential@1": 0.36, - "ndcg_exponential@5": 0.46270632083846136, - "ndcg_exponential@10": 0.49426183883135943, - "precision@1": 0.36, - "precision@5": 0.1104, - "precision@10": 0.065, - "recall@1": 0.36, - "recall@5": 0.552, - "recall@10": 0.65, - "f1@1": 0.36, - "f1@5": 0.18400000000000005, - "f1@10": 0.1181818181818182, - "map@1": 0.36, - "map@5": 0.4329833333333333, - "map@10": 0.4459222222222222, - "mrr@1": 0.36, - "mrr@5": 0.4329833333333333, - "mrr@10": 0.4459222222222222, + "hit@1": 0.387, + "hit@5": 0.582, + "hit@10": 0.666, + "ndcg_linear@1": 0.387, + "ndcg_linear@5": 0.4900446526192341, + "ndcg_linear@10": 0.517109455514934, + "ndcg_exponential@1": 0.387, + "ndcg_exponential@5": 0.4900446526192341, + "ndcg_exponential@10": 0.517109455514934, + "precision@1": 0.387, + "precision@5": 0.11639999999999999, + "precision@10": 0.06659999999999999, + "recall@1": 0.387, + "recall@5": 0.582, + "recall@10": 0.666, + "f1@1": 0.387, + "f1@5": 0.19400000000000006, + "f1@10": 0.12109090909090911, + "map@1": 0.387, + "map@5": 0.45955, + "map@10": 0.47066150793650785, + "mrr@1": 0.387, + "mrr@5": 0.45955, + "mrr@10": 0.47066150793650785, "num_pred": 1000, "num_data": 1000 }, "RefCOCO": { - "hit@1": 0.571, - "hit@5": 0.807, - "hit@10": 0.884, - "ndcg_linear@1": 0.571, - "ndcg_linear@5": 0.6974973204389204, - "ndcg_linear@10": 0.7219755533292225, - "ndcg_exponential@1": 0.571, - "ndcg_exponential@5": 0.6974973204389204, - "ndcg_exponential@10": 0.7219755533292225, - "precision@1": 0.571, - "precision@5": 0.16140000000000002, - "precision@10": 0.0884, - "recall@1": 0.571, - "recall@5": 0.807, - "recall@10": 0.884, - "f1@1": 0.571, - "f1@5": 0.269, - "f1@10": 0.16072727272727275, - "map@1": 0.571, - "map@5": 0.6609833333333334, - "map@10": 0.6708428571428572, - "mrr@1": 0.571, - "mrr@5": 0.6609833333333334, - "mrr@10": 0.6708428571428572, + "hit@1": 0.591, + "hit@5": 0.835, + "hit@10": 0.899, + "ndcg_linear@1": 0.591, + "ndcg_linear@5": 0.7212017026114629, + "ndcg_linear@10": 0.741895085587455, + "ndcg_exponential@1": 0.591, + "ndcg_exponential@5": 0.7212017026114629, + "ndcg_exponential@10": 0.741895085587455, + "precision@1": 0.591, + "precision@5": 0.167, + "precision@10": 0.08990000000000001, + "recall@1": 0.591, + "recall@5": 0.835, + "recall@10": 0.899, + "f1@1": 0.591, + "f1@5": 0.2783333333333334, + "f1@10": 0.16345454545454544, + "map@1": 0.591, + "map@5": 0.6832999999999999, + "map@10": 0.6918392857142858, + "mrr@1": 0.591, + "mrr@5": 0.6832999999999999, + "mrr@10": 0.6918392857142858, "num_pred": 1000, "num_data": 1000 }, "RefCOCO-Matching": { - "hit@1": 0.826, - "hit@5": 0.976, - "hit@10": 0.99, - "ndcg_linear@1": 0.826, - "ndcg_linear@5": 0.9169626654760058, - "ndcg_linear@10": 0.9214732435671692, - "ndcg_exponential@1": 0.826, - "ndcg_exponential@5": 0.9169626654760058, - "ndcg_exponential@10": 0.9214732435671692, - "precision@1": 0.826, - "precision@5": 0.1952, - "precision@10": 0.09900000000000002, - "recall@1": 0.826, - "recall@5": 0.976, - "recall@10": 0.99, - "f1@1": 0.826, - "f1@5": 0.32533333333333336, - "f1@10": 0.17999999999999997, - "map@1": 0.826, - "map@5": 0.8970333333333333, - "map@10": 0.8988869047619048, - "mrr@1": 0.826, - "mrr@5": 0.8940333333333333, - "mrr@10": 0.8958869047619048, + "hit@1": 0.845, + "hit@5": 0.984, + "hit@10": 0.993, + "ndcg_linear@1": 0.845, + "ndcg_linear@5": 0.9307559350032208, + "ndcg_linear@10": 0.9337972586241027, + "ndcg_exponential@1": 0.845, + "ndcg_exponential@5": 0.9307559350032208, + "ndcg_exponential@10": 0.9337972586241027, + "precision@1": 0.845, + "precision@5": 0.19680000000000003, + "precision@10": 0.09930000000000001, + "recall@1": 0.845, + "recall@5": 0.984, + "recall@10": 0.993, + "f1@1": 0.845, + "f1@5": 0.328, + "f1@10": 0.18054545454545454, + "map@1": 0.845, + "map@5": 0.9126833333333334, + "map@10": 0.9140146825396825, + "mrr@1": 0.845, + "mrr@5": 0.9096833333333334, + "mrr@10": 0.9110146825396825, "num_pred": 1000, "num_data": 1000 }, "Visual7W-Pointing": { - "hit@1": 0.512, - "hit@5": 0.746, - "hit@10": 0.821, - "ndcg_linear@1": 0.512, - "ndcg_linear@5": 0.6386876911834943, - "ndcg_linear@10": 0.6627387984931038, - "ndcg_exponential@1": 0.512, - "ndcg_exponential@5": 0.6386876911834943, - "ndcg_exponential@10": 0.6627387984931038, - "precision@1": 0.512, - "precision@5": 0.1492, - "precision@10": 0.08209999999999999, - "recall@1": 0.512, - "recall@5": 0.746, - "recall@10": 0.821, - "f1@1": 0.512, - "f1@5": 0.24866666666666667, - "f1@10": 0.1492727272727273, - "map@1": 0.512, - "map@5": 0.6028166666666667, - "map@10": 0.612631746031746, - "mrr@1": 0.512, - "mrr@5": 0.6028166666666667, - "mrr@10": 0.612631746031746, + "hit@1": 0.515, + "hit@5": 0.75, + "hit@10": 0.836, + "ndcg_linear@1": 0.515, + "ndcg_linear@5": 0.6419346318045233, + "ndcg_linear@10": 0.6697040180599374, + "ndcg_exponential@1": 0.515, + "ndcg_exponential@5": 0.6419346318045233, + "ndcg_exponential@10": 0.6697040180599374, + "precision@1": 0.515, + "precision@5": 0.15, + "precision@10": 0.0836, + "recall@1": 0.515, + "recall@5": 0.75, + "recall@10": 0.836, + "f1@1": 0.515, + "f1@5": 0.25000000000000006, + "f1@10": 0.152, + "map@1": 0.515, + "map@5": 0.6057833333333333, + "map@10": 0.6172190476190477, + "mrr@1": 0.515, + "mrr@5": 0.6057833333333333, + "mrr@10": 0.6172190476190477, "num_pred": 1000, "num_data": 1000 } }, "video": { "K700": { - "hit@1": 0.321, - "hit@5": 0.523, - "hit@10": 0.603, - "ndcg_linear@1": 0.321, - "ndcg_linear@5": 0.4286211523676353, - "ndcg_linear@10": 0.4547881628219443, - "ndcg_exponential@1": 0.321, - "ndcg_exponential@5": 0.4286211523676353, - "ndcg_exponential@10": 0.4547881628219443, - "precision@1": 0.321, - "precision@5": 0.1046, - "precision@10": 0.0603, - "recall@1": 0.321, - "recall@5": 0.523, - "recall@10": 0.603, - "f1@1": 0.321, - "f1@5": 0.17433333333333337, - "f1@10": 0.10963636363636366, - "map@1": 0.321, - "map@5": 0.3971666666666666, - "map@10": 0.4081412698412698, - "mrr@1": 0.321, - "mrr@5": 0.3971666666666666, - "mrr@10": 0.4081412698412698, + "hit@1": 0.322, + "hit@5": 0.531, + "hit@10": 0.605, + "ndcg_linear@1": 0.322, + "ndcg_linear@5": 0.4327176254468425, + "ndcg_linear@10": 0.4565652731906886, + "ndcg_exponential@1": 0.322, + "ndcg_exponential@5": 0.4327176254468425, + "ndcg_exponential@10": 0.4565652731906886, + "precision@1": 0.322, + "precision@5": 0.10619999999999999, + "precision@10": 0.06049999999999999, + "recall@1": 0.322, + "recall@5": 0.531, + "recall@10": 0.605, + "f1@1": 0.322, + "f1@5": 0.17700000000000005, + "f1@10": 0.11000000000000003, + "map@1": 0.322, + "map@5": 0.4000666666666667, + "map@10": 0.40985476190476183, + "mrr@1": 0.322, + "mrr@5": 0.4000666666666667, + "mrr@10": 0.40985476190476183, "num_pred": 1000, "num_data": 1000 }, "SmthSmthV2": { - "hit@1": 0.253, - "hit@5": 0.485, - "hit@10": 0.607, - "ndcg_linear@1": 0.253, - "ndcg_linear@5": 0.37563000406966024, - "ndcg_linear@10": 0.4150475933223388, - "ndcg_exponential@1": 0.253, - "ndcg_exponential@5": 0.37563000406966024, - "ndcg_exponential@10": 0.4150475933223388, - "precision@1": 0.253, - "precision@5": 0.097, - "precision@10": 0.06069999999999999, - "recall@1": 0.253, - "recall@5": 0.485, - "recall@10": 0.607, - "f1@1": 0.253, - "f1@5": 0.1616666666666667, - "f1@10": 0.11036363636363637, - "map@1": 0.253, - "map@5": 0.33931666666666666, - "map@10": 0.3555595238095238, - "mrr@1": 0.253, - "mrr@5": 0.33931666666666666, - "mrr@10": 0.3555595238095238, + "hit@1": 0.255, + "hit@5": 0.487, + "hit@10": 0.602, + "ndcg_linear@1": 0.255, + "ndcg_linear@5": 0.37628929651387355, + "ndcg_linear@10": 0.4135313576823378, + "ndcg_exponential@1": 0.255, + "ndcg_exponential@5": 0.37628929651387355, + "ndcg_exponential@10": 0.4135313576823378, + "precision@1": 0.255, + "precision@5": 0.09739999999999999, + "precision@10": 0.0602, + "recall@1": 0.255, + "recall@5": 0.487, + "recall@10": 0.602, + "f1@1": 0.255, + "f1@5": 0.16233333333333338, + "f1@10": 0.10945454545454547, + "map@1": 0.255, + "map@5": 0.3396666666666667, + "map@10": 0.35507261904761905, + "mrr@1": 0.255, + "mrr@5": 0.3396666666666667, + "mrr@10": 0.35507261904761905, "num_pred": 1000, "num_data": 1000 }, "HMDB51": { - "hit@1": 0.338, - "hit@5": 0.563, - "hit@10": 0.684, - "ndcg_linear@1": 0.338, - "ndcg_linear@5": 0.4553402697715536, - "ndcg_linear@10": 0.4940251051015839, - "ndcg_exponential@1": 0.338, - "ndcg_exponential@5": 0.4553402697715536, - "ndcg_exponential@10": 0.4940251051015839, - "precision@1": 0.338, - "precision@5": 0.11259999999999999, - "precision@10": 0.0684, - "recall@1": 0.338, - "recall@5": 0.563, - "recall@10": 0.684, - "f1@1": 0.338, - "f1@5": 0.18766666666666673, - "f1@10": 0.12436363636363637, - "map@1": 0.338, - "map@5": 0.4197166666666667, - "map@10": 0.4354206349206349, - "mrr@1": 0.338, - "mrr@5": 0.4197166666666667, - "mrr@10": 0.4354206349206349, + "hit@1": 0.344, + "hit@5": 0.568, + "hit@10": 0.693, + "ndcg_linear@1": 0.344, + "ndcg_linear@5": 0.4625676739546081, + "ndcg_linear@10": 0.5028803635848336, + "ndcg_exponential@1": 0.344, + "ndcg_exponential@5": 0.4625676739546081, + "ndcg_exponential@10": 0.5028803635848336, + "precision@1": 0.344, + "precision@5": 0.11359999999999999, + "precision@10": 0.06929999999999999, + "recall@1": 0.344, + "recall@5": 0.568, + "recall@10": 0.693, + "f1@1": 0.344, + "f1@5": 0.1893333333333334, + "f1@10": 0.12600000000000003, + "map@1": 0.344, + "map@5": 0.4274833333333333, + "map@10": 0.4440527777777778, + "mrr@1": 0.344, + "mrr@5": 0.4274833333333333, + "mrr@10": 0.4440527777777778, "num_pred": 1000, "num_data": 1000 }, "UCF101": { - "hit@1": 0.53, - "hit@5": 0.76, - "hit@10": 0.816, - "ndcg_linear@1": 0.53, - "ndcg_linear@5": 0.6535743610111286, - "ndcg_linear@10": 0.6715652735972419, - "ndcg_exponential@1": 0.53, - "ndcg_exponential@5": 0.6535743610111286, - "ndcg_exponential@10": 0.6715652735972419, - "precision@1": 0.53, - "precision@5": 0.152, - "precision@10": 0.08159999999999999, - "recall@1": 0.53, - "recall@5": 0.76, - "recall@10": 0.816, - "f1@1": 0.53, - "f1@5": 0.25333333333333335, - "f1@10": 0.14836363636363636, - "map@1": 0.53, - "map@5": 0.6180666666666667, - "map@10": 0.6254214285714286, - "mrr@1": 0.53, - "mrr@5": 0.6180666666666667, - "mrr@10": 0.6254214285714286, + "hit@1": 0.511, + "hit@5": 0.737, + "hit@10": 0.806, + "ndcg_linear@1": 0.511, + "ndcg_linear@5": 0.6348148772789998, + "ndcg_linear@10": 0.6570765770342009, + "ndcg_exponential@1": 0.511, + "ndcg_exponential@5": 0.6348148772789998, + "ndcg_exponential@10": 0.6570765770342009, + "precision@1": 0.511, + "precision@5": 0.14739999999999998, + "precision@10": 0.0806, + "recall@1": 0.511, + "recall@5": 0.737, + "recall@10": 0.806, + "f1@1": 0.511, + "f1@5": 0.24566666666666673, + "f1@10": 0.14654545454545456, + "map@1": 0.511, + "map@5": 0.6004833333333334, + "map@10": 0.6096404761904762, + "mrr@1": 0.511, + "mrr@5": 0.6004833333333334, + "mrr@10": 0.6096404761904762, "num_pred": 1000, "num_data": 1000 }, "Breakfast": { - "hit@1": 0.20092378752886836, - "hit@5": 0.558891454965358, + "hit@1": 0.19861431870669746, + "hit@5": 0.5842956120092379, "hit@10": 1.0, - "ndcg_linear@1": 0.20092378752886836, - "ndcg_linear@5": 0.3834539592109517, - "ndcg_linear@10": 0.5260901265661807, - "ndcg_exponential@1": 0.20092378752886836, - "ndcg_exponential@5": 0.3834539592109517, - "ndcg_exponential@10": 0.5260901265661807, - "precision@1": 0.20092378752886836, - "precision@5": 0.11177829099307159, + "ndcg_linear@1": 0.19861431870669746, + "ndcg_linear@5": 0.39520539409567434, + "ndcg_linear@10": 0.5290025337835288, + "ndcg_exponential@1": 0.19861431870669746, + "ndcg_exponential@5": 0.39520539409567434, + "ndcg_exponential@10": 0.5290025337835288, + "precision@1": 0.19861431870669746, + "precision@5": 0.11685912240184756, "precision@10": 0.1, - "recall@1": 0.20092378752886836, - "recall@5": 0.558891454965358, + "recall@1": 0.19861431870669746, + "recall@5": 0.5842956120092379, "recall@10": 1.0, - "f1@1": 0.20092378752886836, - "f1@5": 0.18629715165511937, + "f1@1": 0.19861431870669746, + "f1@5": 0.19476520400307934, "f1@10": 0.1818181818181818, - "map@1": 0.20092378752886836, - "map@5": 0.3257505773672056, - "map@10": 0.38463836650903627, - "mrr@1": 0.20092378752886836, - "mrr@5": 0.3257505773672056, - "mrr@10": 0.38463836650903627, + "map@1": 0.19861431870669746, + "map@5": 0.3331023864511163, + "map@10": 0.38796785072766593, + "mrr@1": 0.19861431870669746, + "mrr@5": 0.3331023864511163, + "mrr@10": 0.38796785072766593, "num_pred": 433, "num_data": 433 }, "MVBench": { - "hit@1": 0.37625, + "hit@1": 0.375, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.37625, - "ndcg_linear@5": 0.7166695467315642, - "ndcg_linear@10": 0.7166695467315642, - "ndcg_exponential@1": 0.37625, - "ndcg_exponential@5": 0.7166695467315642, - "ndcg_exponential@10": 0.7166695467315642, - "precision@1": 0.37625, + "ndcg_linear@1": 0.375, + "ndcg_linear@5": 0.7165111050744066, + "ndcg_linear@10": 0.7165111050744066, + "ndcg_exponential@1": 0.375, + "ndcg_exponential@5": 0.7165111050744066, + "ndcg_exponential@10": 0.7165111050744066, + "precision@1": 0.375, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.37625, + "recall@1": 0.375, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.37625, + "f1@1": 0.375, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.37625, - "map@5": 0.6211166666666665, - "map@10": 0.6211166666666665, - "mrr@1": 0.37625, - "mrr@5": 0.6211166666666665, - "mrr@10": 0.6211166666666665, + "map@1": 0.375, + "map@5": 0.620875, + "map@10": 0.620875, + "mrr@1": 0.375, + "mrr@5": 0.620875, + "mrr@10": 0.620875, "num_pred": 4000, "num_data": 4000 }, "Video-MME": { - "hit@1": 0.3511111111111111, + "hit@1": 0.35333333333333333, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.3511111111111111, - "ndcg_linear@5": 0.6953962558796752, - "ndcg_linear@10": 0.6953962558796752, - "ndcg_exponential@1": 0.3511111111111111, - "ndcg_exponential@5": 0.6953962558796752, - "ndcg_exponential@10": 0.6953962558796752, - "precision@1": 0.3511111111111111, + "ndcg_linear@1": 0.35333333333333333, + "ndcg_linear@5": 0.6966614190839566, + "ndcg_linear@10": 0.6966614190839566, + "ndcg_exponential@1": 0.35333333333333333, + "ndcg_exponential@5": 0.6966614190839566, + "ndcg_exponential@10": 0.6966614190839566, + "precision@1": 0.35333333333333333, "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, - "recall@1": 0.3511111111111111, + "recall@1": 0.35333333333333333, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.3511111111111111, + "f1@1": 0.35333333333333333, "f1@5": 0.3333333333333334, "f1@10": 0.18181818181818188, - "map@1": 0.3511111111111111, - "map@5": 0.5934876543209876, - "map@10": 0.5934876543209876, - "mrr@1": 0.3511111111111111, - "mrr@5": 0.5934876543209876, - "mrr@10": 0.5934876543209876, + "map@1": 0.35333333333333333, + "map@5": 0.5951543209876543, + "map@10": 0.5951543209876543, + "mrr@1": 0.35333333333333333, + "mrr@5": 0.5951543209876543, + "mrr@10": 0.5951543209876543, "num_pred": 2700, "num_data": 2700 }, "NExTQA": { - "hit@1": 0.44932274638019615, + "hit@1": 0.44990658570761327, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.44932274638019615, - "ndcg_linear@5": 0.732425356090162, - "ndcg_linear@10": 0.732425356090162, - "ndcg_exponential@1": 0.44932274638019615, - "ndcg_exponential@5": 0.732425356090162, - "ndcg_exponential@10": 0.732425356090162, - "precision@1": 0.44932274638019615, + "ndcg_linear@1": 0.44990658570761327, + "ndcg_linear@5": 0.7326473160283148, + "ndcg_linear@10": 0.7326473160283148, + "ndcg_exponential@1": 0.44990658570761327, + "ndcg_exponential@5": 0.7326473160283148, + "ndcg_exponential@10": 0.7326473160283148, + "precision@1": 0.44990658570761327, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.44932274638019615, + "recall@1": 0.44990658570761327, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.44932274638019615, + "f1@1": 0.44990658570761327, "f1@5": 0.3333333333333333, "f1@10": 0.1818181818181818, - "map@1": 0.44932274638019615, - "map@5": 0.6443542737038767, - "map@10": 0.6443542737038767, - "mrr@1": 0.44932274638019615, - "mrr@5": 0.6441596605947376, - "mrr@10": 0.6441596605947376, + "map@1": 0.44990658570761327, + "map@5": 0.6446500856297681, + "map@10": 0.6446500856297681, + "mrr@1": 0.44990658570761327, + "mrr@5": 0.6444554725206291, + "mrr@10": 0.6444554725206291, "num_pred": 8564, "num_data": 8564 }, "EgoSchema": { - "hit@1": 0.47, + "hit@1": 0.472, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.47, - "ndcg_linear@5": 0.7542281638188373, - "ndcg_linear@10": 0.7542281638188373, - "ndcg_exponential@1": 0.47, - "ndcg_exponential@5": 0.7542281638188373, - "ndcg_exponential@10": 0.7542281638188373, - "precision@1": 0.47, + "ndcg_linear@1": 0.472, + "ndcg_linear@5": 0.752609568747408, + "ndcg_linear@10": 0.752609568747408, + "ndcg_exponential@1": 0.472, + "ndcg_exponential@5": 0.752609568747408, + "ndcg_exponential@10": 0.752609568747408, + "precision@1": 0.472, "precision@5": 0.2, "precision@10": 0.1, - "recall@1": 0.47, + "recall@1": 0.472, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.47, + "f1@1": 0.472, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.47, - "map@5": 0.6721333333333334, - "map@10": 0.6721333333333334, - "mrr@1": 0.47, - "mrr@5": 0.6721333333333334, - "mrr@10": 0.6721333333333334, + "map@1": 0.472, + "map@5": 0.6701333333333334, + "map@10": 0.6701333333333334, + "mrr@1": 0.472, + "mrr@5": 0.6701333333333334, + "mrr@10": 0.6701333333333334, "num_pred": 500, "num_data": 500 }, @@ -1299,226 +1299,254 @@ "num_data": 1000 }, "DiDeMo": { - "hit@1": 0.22808764940239043, - "hit@5": 0.44721115537848605, - "hit@10": 0.5687250996015937, - "ndcg_linear@1": 0.22808764940239043, - "ndcg_linear@5": 0.3427140320884816, - "ndcg_linear@10": 0.3822101935689942, - "ndcg_exponential@1": 0.22808764940239043, - "ndcg_exponential@5": 0.3427140320884816, - "ndcg_exponential@10": 0.3822101935689942, - "precision@1": 0.22808764940239043, - "precision@5": 0.08944223107569721, - "precision@10": 0.056872509960159356, - "recall@1": 0.22808764940239043, - "recall@5": 0.44721115537848605, - "recall@10": 0.5687250996015937, - "f1@1": 0.22808764940239043, - "f1@5": 0.14907038512616205, - "f1@10": 0.10340456356392612, - "map@1": 0.22808764940239043, - "map@5": 0.30810092961487384, - "map@10": 0.32451859229747676, - "mrr@1": 0.22808764940239043, - "mrr@5": 0.30810092961487384, - "mrr@10": 0.32451859229747676, + "hit@1": 0.23705179282868527, + "hit@5": 0.450199203187251, + "hit@10": 0.5527888446215139, + "ndcg_linear@1": 0.23705179282868527, + "ndcg_linear@5": 0.34867187665048255, + "ndcg_linear@10": 0.382378756025927, + "ndcg_exponential@1": 0.23705179282868527, + "ndcg_exponential@5": 0.34867187665048255, + "ndcg_exponential@10": 0.382378756025927, + "precision@1": 0.23705179282868527, + "precision@5": 0.0900398406374502, + "precision@10": 0.05527888446215139, + "recall@1": 0.23705179282868527, + "recall@5": 0.450199203187251, + "recall@10": 0.5527888446215139, + "f1@1": 0.23705179282868527, + "f1@5": 0.15006640106241706, + "f1@10": 0.1005070626584571, + "map@1": 0.23705179282868527, + "map@5": 0.3150564409030545, + "map@10": 0.32927575412635174, + "mrr@1": 0.23705179282868527, + "mrr@5": 0.3150564409030545, + "mrr@10": 0.32927575412635174, "num_pred": 1004, "num_data": 1004 }, "MSR-VTT": { - "hit@1": 0.25, - "hit@5": 0.474, - "hit@10": 0.576, - "ndcg_linear@1": 0.25, - "ndcg_linear@5": 0.3652809849605177, - "ndcg_linear@10": 0.3983339947967488, - "ndcg_exponential@1": 0.25, - "ndcg_exponential@5": 0.3652809849605177, - "ndcg_exponential@10": 0.3983339947967488, - "precision@1": 0.25, - "precision@5": 0.09479999999999998, - "precision@10": 0.05759999999999999, - "recall@1": 0.25, - "recall@5": 0.474, - "recall@10": 0.576, - "f1@1": 0.25, - "f1@5": 0.15800000000000006, - "f1@10": 0.10472727272727275, - "map@1": 0.25, - "map@5": 0.3294666666666667, - "map@10": 0.3431484126984127, - "mrr@1": 0.25, - "mrr@5": 0.3294666666666667, - "mrr@10": 0.3431484126984127, + "hit@1": 0.259, + "hit@5": 0.488, + "hit@10": 0.577, + "ndcg_linear@1": 0.259, + "ndcg_linear@5": 0.3759756863463764, + "ndcg_linear@10": 0.4050712623005791, + "ndcg_exponential@1": 0.259, + "ndcg_exponential@5": 0.3759756863463764, + "ndcg_exponential@10": 0.4050712623005791, + "precision@1": 0.259, + "precision@5": 0.09759999999999999, + "precision@10": 0.057699999999999994, + "recall@1": 0.259, + "recall@5": 0.488, + "recall@10": 0.577, + "f1@1": 0.259, + "f1@5": 0.1626666666666667, + "f1@10": 0.10490909090909094, + "map@1": 0.259, + "map@5": 0.33913333333333334, + "map@10": 0.3513257936507937, + "mrr@1": 0.259, + "mrr@5": 0.33913333333333334, + "mrr@10": 0.3513257936507937, "num_pred": 1000, "num_data": 1000 }, "MSVD": { - "hit@1": 0.41940298507462687, - "hit@5": 0.7, - "hit@10": 0.7805970149253731, - "ndcg_linear@1": 0.41940298507462687, - "ndcg_linear@5": 0.5692519862375854, - "ndcg_linear@10": 0.5955843294718209, - "ndcg_exponential@1": 0.41940298507462687, - "ndcg_exponential@5": 0.5692519862375854, - "ndcg_exponential@10": 0.5955843294718209, - "precision@1": 0.41940298507462687, - "precision@5": 0.13999999999999999, - "precision@10": 0.07805970149253731, - "recall@1": 0.41940298507462687, - "recall@5": 0.7, - "recall@10": 0.7805970149253731, - "f1@1": 0.41940298507462687, - "f1@5": 0.2333333333333334, - "f1@10": 0.14192672998643152, - "map@1": 0.41940298507462687, - "map@5": 0.5256965174129353, - "map@10": 0.5367264866145464, - "mrr@1": 0.41940298507462687, - "mrr@5": 0.5256965174129353, - "mrr@10": 0.5367264866145464, + "hit@1": 0.42835820895522386, + "hit@5": 0.7029850746268657, + "hit@10": 0.7910447761194029, + "ndcg_linear@1": 0.42835820895522386, + "ndcg_linear@5": 0.5745704742800433, + "ndcg_linear@10": 0.6033343607584918, + "ndcg_exponential@1": 0.42835820895522386, + "ndcg_exponential@5": 0.5745704742800433, + "ndcg_exponential@10": 0.6033343607584918, + "precision@1": 0.42835820895522386, + "precision@5": 0.1405970149253731, + "precision@10": 0.07910447761194028, + "recall@1": 0.42835820895522386, + "recall@5": 0.7029850746268657, + "recall@10": 0.7910447761194029, + "f1@1": 0.42835820895522386, + "f1@5": 0.23432835820895526, + "f1@10": 0.14382632293080055, + "map@1": 0.42835820895522386, + "map@5": 0.5318407960199005, + "map@10": 0.5438811892916371, + "mrr@1": 0.42835820895522386, + "mrr@5": 0.5318407960199005, + "mrr@10": 0.5438811892916371, "num_pred": 670, "num_data": 670 }, "VATEX": { - "hit@1": 0.1864671728450201, - "hit@5": 0.39794551138901296, - "hit@10": 0.49933005806163466, - "ndcg_linear@1": 0.1864671728450201, - "ndcg_linear@5": 0.2965339753650619, - "ndcg_linear@10": 0.32929416685693125, - "ndcg_exponential@1": 0.1864671728450201, - "ndcg_exponential@5": 0.2965339753650619, - "ndcg_exponential@10": 0.32929416685693125, - "precision@1": 0.1864671728450201, - "precision@5": 0.0795891022778026, - "precision@10": 0.04993300580616347, - "recall@1": 0.1864671728450201, - "recall@5": 0.39794551138901296, - "recall@10": 0.49933005806163466, - "f1@1": 0.1864671728450201, - "f1@5": 0.13264850379633766, - "f1@10": 0.09078728328393358, - "map@1": 0.1864671728450201, - "map@5": 0.2630005955039452, - "map@10": 0.2765067490447124, - "mrr@1": 0.1864671728450201, - "mrr@5": 0.2630005955039452, - "mrr@10": 0.2765067490447124, + "hit@1": 0.18289414917373828, + "hit@5": 0.4015185350602948, + "hit@10": 0.49977668602054487, + "ndcg_linear@1": 0.18289414917373828, + "ndcg_linear@5": 0.2964472286355593, + "ndcg_linear@10": 0.328319269909704, + "ndcg_exponential@1": 0.18289414917373828, + "ndcg_exponential@5": 0.2964472286355593, + "ndcg_exponential@10": 0.328319269909704, + "precision@1": 0.18289414917373828, + "precision@5": 0.08030370701205897, + "precision@10": 0.04997766860205449, + "recall@1": 0.18289414917373828, + "recall@5": 0.4015185350602948, + "recall@10": 0.49977668602054487, + "f1@1": 0.18289414917373828, + "f1@5": 0.13383951168676494, + "f1@10": 0.0908684883673718, + "map@1": 0.18289414917373828, + "map@5": 0.2617388715200238, + "map@10": 0.27494895680469594, + "mrr@1": 0.18289414917373828, + "mrr@5": 0.2617388715200238, + "mrr@10": 0.27494895680469594, "num_pred": 4478, "num_data": 4478 }, "YouCook2": { - "hit@1": 0.07518087448883297, - "hit@5": 0.18748033972947467, - "hit@10": 0.2560553633217993, - "ndcg_linear@1": 0.07518087448883297, - "ndcg_linear@5": 0.13240906083348206, - "ndcg_linear@10": 0.15449506875832897, - "ndcg_exponential@1": 0.07518087448883297, - "ndcg_exponential@5": 0.13240906083348206, - "ndcg_exponential@10": 0.15449506875832897, - "precision@1": 0.07518087448883297, - "precision@5": 0.03749606794589494, - "precision@10": 0.02560553633217993, - "recall@1": 0.07518087448883297, - "recall@5": 0.18748033972947467, - "recall@10": 0.2560553633217993, - "f1@1": 0.07518087448883297, - "f1@5": 0.062493446576491564, - "f1@10": 0.04655552060396351, - "map@1": 0.07518087448883297, - "map@5": 0.11430743420362798, - "map@10": 0.12336900892264213, - "mrr@1": 0.07518087448883297, - "mrr@5": 0.11430743420362798, - "mrr@10": 0.12336900892264213, + "hit@1": 0.0764391318024536, + "hit@5": 0.18842403271469016, + "hit@10": 0.2626612142183076, + "ndcg_linear@1": 0.0764391318024536, + "ndcg_linear@5": 0.13367218143274567, + "ndcg_linear@10": 0.15750022399206454, + "ndcg_exponential@1": 0.0764391318024536, + "ndcg_exponential@5": 0.13367218143274567, + "ndcg_exponential@10": 0.15750022399206454, + "precision@1": 0.0764391318024536, + "precision@5": 0.037684806542938036, + "precision@10": 0.026266121421830765, + "recall@1": 0.0764391318024536, + "recall@5": 0.18842403271469016, + "recall@10": 0.2626612142183076, + "f1@1": 0.0764391318024536, + "f1@5": 0.06280801090489672, + "f1@10": 0.04775658440332866, + "map@1": 0.0764391318024536, + "map@5": 0.11567578903219042, + "map@10": 0.1254096825896134, + "mrr@1": 0.0764391318024536, + "mrr@5": 0.11567578903219042, + "mrr@10": 0.1254096825896134, "num_pred": 3179, "num_data": 3179 }, "QVHighlight": { - "hit@1": 0.6084949215143121, - "hit@5": 0.8928901200369345, + "hit@1": 0.615881809787627, + "hit@5": 0.9021237303785781, "hit@10": 1.0, - "ndcg_linear@1": 0.6084949215143121, - "ndcg_linear@5": 0.7589804905293247, - "ndcg_linear@10": 0.7936722925896575, - "ndcg_exponential@1": 0.6084949215143121, - "ndcg_exponential@5": 0.7589804905293247, - "ndcg_exponential@10": 0.7936722925896575, - "precision@1": 0.6084949215143121, - "precision@5": 0.17857802400738687, + "ndcg_linear@1": 0.615881809787627, + "ndcg_linear@5": 0.7645615723857577, + "ndcg_linear@10": 0.7960967642135737, + "ndcg_exponential@1": 0.615881809787627, + "ndcg_exponential@5": 0.7645615723857577, + "ndcg_exponential@10": 0.7960967642135737, + "precision@1": 0.615881809787627, + "precision@5": 0.1804247460757156, "precision@10": 0.09999999999999998, - "recall@1": 0.6084949215143121, - "recall@5": 0.8928901200369345, + "recall@1": 0.615881809787627, + "recall@5": 0.9021237303785781, "recall@10": 1.0, - "f1@1": 0.6084949215143121, - "f1@5": 0.2976300400123115, + "f1@1": 0.615881809787627, + "f1@5": 0.30070791012619275, "f1@10": 0.18181818181818185, - "map@1": 0.6084949215143121, - "map@5": 0.7144967682363804, - "map@10": 0.7288473376423515, - "mrr@1": 0.6084949215143121, - "mrr@5": 0.7144967682363804, - "mrr@10": 0.7288473376423515, + "map@1": 0.615881809787627, + "map@5": 0.7190827947060634, + "map@10": 0.7320296354922394, + "mrr@1": 0.615881809787627, + "mrr@5": 0.7190827947060634, + "mrr@10": 0.7320296354922394, "num_pred": 1083, "num_data": 1083 }, "Charades-STA": { - "hit@1": 0.18844566712517194, - "hit@5": 0.5722145804676754, + "hit@1": 0.18982118294360384, + "hit@5": 0.5433287482806052, "hit@10": 1.0, - "ndcg_linear@1": 0.18844566712517194, - "ndcg_linear@5": 0.37699818670569085, - "ndcg_linear@10": 0.5119551257659812, - "ndcg_exponential@1": 0.18844566712517194, - "ndcg_exponential@5": 0.37699818670569085, - "ndcg_exponential@10": 0.5119551257659812, - "precision@1": 0.18844566712517194, - "precision@5": 0.11444291609353506, + "ndcg_linear@1": 0.18982118294360384, + "ndcg_linear@5": 0.365216628088626, + "ndcg_linear@10": 0.5099058697938778, + "ndcg_exponential@1": 0.18982118294360384, + "ndcg_exponential@5": 0.365216628088626, + "ndcg_exponential@10": 0.5099058697938778, + "precision@1": 0.18982118294360384, + "precision@5": 0.10866574965612105, "precision@10": 0.09999999999999998, - "recall@1": 0.18844566712517194, - "recall@5": 0.5722145804676754, + "recall@1": 0.18982118294360384, + "recall@5": 0.5433287482806052, "recall@10": 1.0, - "f1@1": 0.18844566712517194, - "f1@5": 0.1907381934892252, + "f1@1": 0.18982118294360384, + "f1@5": 0.18110958276020178, "f1@10": 0.18181818181818185, - "map@1": 0.18844566712517194, - "map@5": 0.31364053186611646, - "map@10": 0.3674133752538154, - "mrr@1": 0.18844566712517194, - "mrr@5": 0.31364053186611646, - "mrr@10": 0.3674133752538154, + "map@1": 0.18982118294360384, + "map@5": 0.3072443833104081, + "map@10": 0.36524530032095365, + "mrr@1": 0.18982118294360384, + "mrr@5": 0.3072443833104081, + "mrr@10": 0.36524530032095365, "num_pred": 727, "num_data": 727 }, "MomentSeeker": { - "hit@1": 0.31777777777777777, - "hit@5": 0.8033333333333333, + "hit@1": 0.3333333333333333, + "hit@5": 0.8083645443196005, "hit@10": 1.0, - "ndcg_linear@1": 0.31777777777777777, - "ndcg_linear@5": 0.5664993737348916, - "ndcg_linear@10": 0.6381761228963511, - "ndcg_exponential@1": 0.31777777777777777, - "ndcg_exponential@5": 0.5664993737348916, - "ndcg_exponential@10": 0.6381761228963511, - "precision@1": 0.31777777777777777, - "precision@5": 0.1767777777777778, + "ndcg_linear@1": 0.3333333333333333, + "ndcg_linear@5": 0.5752977935601024, + "ndcg_linear@10": 0.6460130742488344, + "ndcg_exponential@1": 0.3333333333333333, + "ndcg_exponential@5": 0.5752977935601024, + "ndcg_exponential@10": 0.6460130742488344, + "precision@1": 0.3333333333333333, + "precision@5": 0.1795255930087391, + "precision@10": 0.11460674157303372, + "recall@1": 0.30703807740324596, + "recall@5": 0.7879473574698292, + "recall@10": 0.9985955056179775, + "f1@1": 0.3149059707112141, + "f1@5": 0.28690445357112027, + "f1@10": 0.20263084120762023, + "map@1": 0.3333333333333333, + "map@5": 0.4983241434318213, + "map@10": 0.5305713571494627, + "mrr@1": 0.3333333333333333, + "mrr@5": 0.5108510195588847, + "mrr@10": 0.5368770188058578, + "num_pred": 1602, + "num_data": 1602 + }, + "MomentSeeker_1k8": { + "hit@1": 0.3161111111111111, + "hit@5": 0.805, + "hit@10": 1.0, + "ndcg_linear@1": 0.3161111111111111, + "ndcg_linear@5": 0.565052316885336, + "ndcg_linear@10": 0.6360071566647605, + "ndcg_exponential@1": 0.3161111111111111, + "ndcg_exponential@5": 0.565052316885336, + "ndcg_exponential@10": 0.6360071566647605, + "precision@1": 0.3161111111111111, + "precision@5": 0.17700000000000002, "precision@10": 0.11305555555555558, - "recall@1": 0.29465277777777776, - "recall@5": 0.7851620370370369, + "recall@1": 0.29270833333333335, + "recall@5": 0.7868287037037037, "recall@10": 0.99875, - "f1@1": 0.30156261022927683, - "f1@5": 0.2835724769058103, + "f1@1": 0.299710758377425, + "f1@5": 0.2839957573290907, "f1@10": 0.2004256237589571, - "map@1": 0.31777777777777777, - "map@5": 0.4882804012345679, - "map@10": 0.5207349077748383, - "mrr@1": 0.31777777777777777, - "mrr@5": 0.49910185185185185, - "mrr@10": 0.5260895061728394, + "map@1": 0.3161111111111111, + "map@5": 0.4858881172839506, + "map@10": 0.5179637988683128, + "mrr@1": 0.3161111111111111, + "mrr@5": 0.4969907407407407, + "mrr@10": 0.5235295414462081, "num_pred": 1800, "num_data": 1800 } @@ -1527,252 +1555,252 @@ "ViDoRe_arxivqa": { "hit@1": 0.45, "hit@5": 0.602, - "hit@10": 0.67, + "hit@10": 0.666, "ndcg_linear@1": 0.45, - "ndcg_linear@5": 0.5298461278515469, - "ndcg_linear@10": 0.5513653011991692, + "ndcg_linear@5": 0.53040179838485, + "ndcg_linear@10": 0.5509894618901121, "ndcg_exponential@1": 0.45, - "ndcg_exponential@5": 0.5298461278515469, - "ndcg_exponential@10": 0.5513653011991692, + "ndcg_exponential@5": 0.53040179838485, + "ndcg_exponential@10": 0.5509894618901121, "precision@1": 0.45, "precision@5": 0.1204, - "precision@10": 0.067, + "precision@10": 0.06659999999999999, "recall@1": 0.45, "recall@5": 0.602, - "recall@10": 0.67, + "recall@10": 0.666, "f1@1": 0.45, - "f1@5": 0.20066666666666674, - "f1@10": 0.12181818181818184, + "f1@5": 0.20066666666666672, + "f1@10": 0.12109090909090911, "map@1": 0.45, - "map@5": 0.5059666666666667, - "map@10": 0.5145904761904763, + "map@5": 0.5066, + "map@10": 0.5150452380952382, "mrr@1": 0.45, - "mrr@5": 0.5059666666666667, - "mrr@10": 0.5145904761904763, + "mrr@5": 0.5066, + "mrr@10": 0.5150452380952382, "num_pred": 500, "num_data": 500 }, "ViDoRe_docvqa": { - "hit@1": 0.15964523281596452, + "hit@1": 0.1574279379157428, "hit@5": 0.34146341463414637, - "hit@10": 0.4279379157427938, - "ndcg_linear@1": 0.15964523281596452, - "ndcg_linear@5": 0.2541845458267961, - "ndcg_linear@10": 0.2816434324462572, - "ndcg_exponential@1": 0.15964523281596452, - "ndcg_exponential@5": 0.2541845458267961, - "ndcg_exponential@10": 0.2816434324462572, - "precision@1": 0.15964523281596452, + "hit@10": 0.44124168514412415, + "ndcg_linear@1": 0.1574279379157428, + "ndcg_linear@5": 0.25419153797828753, + "ndcg_linear@10": 0.28596454486335576, + "ndcg_exponential@1": 0.1574279379157428, + "ndcg_exponential@5": 0.25419153797828753, + "ndcg_exponential@10": 0.28596454486335576, + "precision@1": 0.1574279379157428, "precision@5": 0.06962305986696231, - "precision@10": 0.043458980044345896, - "recall@1": 0.15787139689578714, + "precision@10": 0.04478935698447893, + "recall@1": 0.15565410199556542, "recall@5": 0.3356984478935698, - "recall@10": 0.42050997782705096, - "f1@1": 0.15816703621581674, + "recall@10": 0.43381374722838134, + "f1@1": 0.155949741315595, "f1@5": 0.11447048886073279, - "f1@10": 0.07839448061177551, - "map@1": 0.15964523281596452, - "map@5": 0.22555432372505546, - "map@10": 0.23678421849153558, - "mrr@1": 0.15964523281596452, + "f1@10": 0.08081334777565377, + "map@1": 0.1574279379157428, + "map@5": 0.22548041389504805, + "map@10": 0.23851582022313728, + "mrr@1": 0.1574279379157428, "mrr@5": 0.22893569844789358, - "mrr@10": 0.24044275507690144, + "mrr@10": 0.24224826663851054, "num_pred": 451, "num_data": 451 }, "ViDoRe_infovqa": { - "hit@1": 0.6133603238866396, + "hit@1": 0.611336032388664, "hit@5": 0.8178137651821862, - "hit@10": 0.8805668016194332, - "ndcg_linear@1": 0.6133603238866396, - "ndcg_linear@5": 0.7231460909674565, - "ndcg_linear@10": 0.7438877002962954, - "ndcg_exponential@1": 0.6133603238866396, - "ndcg_exponential@5": 0.7231460909674565, - "ndcg_exponential@10": 0.7438877002962954, - "precision@1": 0.6133603238866396, + "hit@10": 0.8866396761133604, + "ndcg_linear@1": 0.611336032388664, + "ndcg_linear@5": 0.7219049023181336, + "ndcg_linear@10": 0.7443800711931109, + "ndcg_exponential@1": 0.611336032388664, + "ndcg_exponential@5": 0.7219049023181336, + "ndcg_exponential@10": 0.7443800711931109, + "precision@1": 0.611336032388664, "precision@5": 0.16356275303643722, - "precision@10": 0.08805668016194332, - "recall@1": 0.6123481781376519, + "precision@10": 0.08866396761133602, + "recall@1": 0.6103238866396761, "recall@5": 0.8168016194331984, - "recall@10": 0.8795546558704453, - "f1@1": 0.612685560053981, + "recall@10": 0.8856275303643725, + "f1@1": 0.6106612685560053, "f1@5": 0.2725081935608252, - "f1@10": 0.1600723837565943, - "map@1": 0.6133603238866396, - "map@5": 0.6914642375168691, - "map@10": 0.7002907910802648, - "mrr@1": 0.6133603238866396, - "mrr@5": 0.692476383265857, - "mrr@10": 0.7013029368292527, + "f1@10": 0.16117654275549012, + "map@1": 0.611336032388664, + "map@5": 0.6898448043184885, + "map@10": 0.6992473170104749, + "mrr@1": 0.611336032388664, + "mrr@5": 0.6908569500674763, + "mrr@10": 0.7002594627594628, "num_pred": 494, "num_data": 494 }, "ViDoRe_tabfquad": { - "hit@1": 0.5428571428571428, - "hit@5": 0.7607142857142857, - "hit@10": 0.8142857142857143, - "ndcg_linear@1": 0.5428571428571428, - "ndcg_linear@5": 0.6607864639631176, - "ndcg_linear@10": 0.6777435779728691, - "ndcg_exponential@1": 0.5428571428571428, - "ndcg_exponential@5": 0.6607864639631176, - "ndcg_exponential@10": 0.6777435779728691, - "precision@1": 0.5428571428571428, - "precision@5": 0.15214285714285714, - "precision@10": 0.08142857142857142, - "recall@1": 0.5428571428571428, - "recall@5": 0.7607142857142857, - "recall@10": 0.8142857142857143, - "f1@1": 0.5428571428571428, - "f1@5": 0.2535714285714286, - "f1@10": 0.14805194805194807, - "map@1": 0.5428571428571428, - "map@5": 0.6272619047619047, - "map@10": 0.634044784580499, - "mrr@1": 0.5428571428571428, - "mrr@5": 0.6272619047619047, - "mrr@10": 0.634044784580499, + "hit@1": 0.55, + "hit@5": 0.7571428571428571, + "hit@10": 0.8035714285714286, + "ndcg_linear@1": 0.55, + "ndcg_linear@5": 0.665064789033657, + "ndcg_linear@10": 0.6799445607243767, + "ndcg_exponential@1": 0.55, + "ndcg_exponential@5": 0.665064789033657, + "ndcg_exponential@10": 0.6799445607243767, + "precision@1": 0.55, + "precision@5": 0.1514285714285714, + "precision@10": 0.08035714285714286, + "recall@1": 0.55, + "recall@5": 0.7571428571428571, + "recall@10": 0.8035714285714286, + "f1@1": 0.55, + "f1@5": 0.25238095238095243, + "f1@10": 0.14610389610389612, + "map@1": 0.55, + "map@5": 0.6340476190476191, + "map@10": 0.6401119614512472, + "mrr@1": 0.55, + "mrr@5": 0.6340476190476191, + "mrr@10": 0.6401119614512472, "num_pred": 280, "num_data": 280 }, "ViDoRe_tatdqa": { - "hit@1": 0.15674362089914945, - "hit@5": 0.35236938031591736, - "hit@10": 0.4641555285540705, - "ndcg_linear@1": 0.15674362089914945, - "ndcg_linear@5": 0.2585284591791044, - "ndcg_linear@10": 0.29467381069418136, - "ndcg_exponential@1": 0.15674362089914945, - "ndcg_exponential@5": 0.2585284591791044, - "ndcg_exponential@10": 0.29467381069418136, - "precision@1": 0.15674362089914945, - "precision@5": 0.07059538274605104, - "precision@10": 0.04659781287970838, - "recall@1": 0.15674362089914945, - "recall@5": 0.3515188335358445, - "recall@10": 0.46366950182260025, - "f1@1": 0.15674362089914945, - "f1@5": 0.11745067407278832, - "f1@10": 0.08458706137928496, - "map@1": 0.15674362089914945, - "map@5": 0.2275567031186715, - "map@10": 0.24240993076047754, - "mrr@1": 0.15674362089914945, - "mrr@5": 0.22790603483191574, - "mrr@10": 0.24270863468919363, + "hit@1": 0.15613608748481167, + "hit@5": 0.3572296476306197, + "hit@10": 0.4617253948967193, + "ndcg_linear@1": 0.15613608748481167, + "ndcg_linear@5": 0.2601005410907324, + "ndcg_linear@10": 0.2936653233860738, + "ndcg_exponential@1": 0.15613608748481167, + "ndcg_exponential@5": 0.2601005410907324, + "ndcg_exponential@10": 0.2936653233860738, + "precision@1": 0.15613608748481167, + "precision@5": 0.0715674362089915, + "precision@10": 0.04635479951397327, + "recall@1": 0.15613608748481167, + "recall@5": 0.3563791008505468, + "recall@10": 0.46123936816524913, + "f1@1": 0.15613608748481167, + "f1@5": 0.11907076317768907, + "f1@10": 0.08414521889613019, + "map@1": 0.15613608748481167, + "map@5": 0.2281136087484812, + "map@10": 0.2417688865744759, + "mrr@1": 0.15613608748481167, + "mrr@5": 0.2284629404617254, + "mrr@10": 0.24206084013192156, "num_pred": 1646, "num_data": 1646 }, "ViDoRe_shiftproject": { - "hit@1": 0.13, - "hit@5": 0.4, - "hit@10": 0.6, - "ndcg_linear@1": 0.13, - "ndcg_linear@5": 0.2734127450408923, - "ndcg_linear@10": 0.33883350664214873, - "ndcg_exponential@1": 0.13, - "ndcg_exponential@5": 0.2734127450408923, - "ndcg_exponential@10": 0.33883350664214873, - "precision@1": 0.13, - "precision@5": 0.08, - "precision@10": 0.059999999999999984, - "recall@1": 0.13, - "recall@5": 0.4, - "recall@10": 0.6, - "f1@1": 0.13, - "f1@5": 0.1333333333333334, - "f1@10": 0.1090909090909091, - "map@1": 0.13, - "map@5": 0.23133333333333334, - "map@10": 0.25872222222222224, - "mrr@1": 0.13, - "mrr@5": 0.23133333333333334, - "mrr@10": 0.25872222222222224, + "hit@1": 0.14, + "hit@5": 0.41, + "hit@10": 0.59, + "ndcg_linear@1": 0.14, + "ndcg_linear@5": 0.2813330417830941, + "ndcg_linear@10": 0.3398083442076662, + "ndcg_exponential@1": 0.14, + "ndcg_exponential@5": 0.2813330417830941, + "ndcg_exponential@10": 0.3398083442076662, + "precision@1": 0.14, + "precision@5": 0.08199999999999999, + "precision@10": 0.05899999999999998, + "recall@1": 0.14, + "recall@5": 0.41, + "recall@10": 0.59, + "f1@1": 0.14, + "f1@5": 0.13666666666666671, + "f1@10": 0.10727272727272728, + "map@1": 0.14, + "map@5": 0.23883333333333334, + "map@10": 0.2630674603174603, + "mrr@1": 0.14, + "mrr@5": 0.23883333333333334, + "mrr@10": 0.2630674603174603, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.59, - "hit@5": 0.85, - "hit@10": 0.88, - "ndcg_linear@1": 0.59, - "ndcg_linear@5": 0.7201422053206411, - "ndcg_linear@10": 0.7299635207551652, - "ndcg_exponential@1": 0.59, - "ndcg_exponential@5": 0.7201422053206411, - "ndcg_exponential@10": 0.7299635207551652, - "precision@1": 0.59, - "precision@5": 0.16999999999999996, - "precision@10": 0.08799999999999997, - "recall@1": 0.59, - "recall@5": 0.85, - "recall@10": 0.88, - "f1@1": 0.59, - "f1@5": 0.2833333333333334, - "f1@10": 0.16000000000000003, - "map@1": 0.59, - "map@5": 0.6778333333333333, - "map@10": 0.681940476190476, - "mrr@1": 0.59, - "mrr@5": 0.6778333333333333, - "mrr@10": 0.681940476190476, + "hit@1": 0.57, + "hit@5": 0.83, + "hit@10": 0.89, + "ndcg_linear@1": 0.57, + "ndcg_linear@5": 0.7030983835952165, + "ndcg_linear@10": 0.7227057524973328, + "ndcg_exponential@1": 0.57, + "ndcg_exponential@5": 0.7030983835952165, + "ndcg_exponential@10": 0.7227057524973328, + "precision@1": 0.57, + "precision@5": 0.16599999999999998, + "precision@10": 0.08899999999999997, + "recall@1": 0.57, + "recall@5": 0.83, + "recall@10": 0.89, + "f1@1": 0.57, + "f1@5": 0.2766666666666667, + "f1@10": 0.16181818181818186, + "map@1": 0.57, + "map@5": 0.6613333333333332, + "map@10": 0.6695357142857142, + "mrr@1": 0.57, + "mrr@5": 0.6613333333333332, + "mrr@10": 0.6695357142857142, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.52, - "hit@5": 0.76, - "hit@10": 0.85, - "ndcg_linear@1": 0.52, - "ndcg_linear@5": 0.6517245745315876, - "ndcg_linear@10": 0.6815359848536943, - "ndcg_exponential@1": 0.52, - "ndcg_exponential@5": 0.6517245745315876, - "ndcg_exponential@10": 0.6815359848536943, - "precision@1": 0.52, - "precision@5": 0.15199999999999997, - "precision@10": 0.08499999999999998, - "recall@1": 0.52, - "recall@5": 0.76, - "recall@10": 0.85, - "f1@1": 0.52, - "f1@5": 0.25333333333333335, - "f1@10": 0.15454545454545457, - "map@1": 0.52, - "map@5": 0.6154999999999999, - "map@10": 0.6281984126984126, - "mrr@1": 0.52, - "mrr@5": 0.6154999999999999, - "mrr@10": 0.6281984126984126, + "hit@1": 0.54, + "hit@5": 0.78, + "hit@10": 0.86, + "ndcg_linear@1": 0.54, + "ndcg_linear@5": 0.6633479655166431, + "ndcg_linear@10": 0.6888315855208215, + "ndcg_exponential@1": 0.54, + "ndcg_exponential@5": 0.6633479655166431, + "ndcg_exponential@10": 0.6888315855208215, + "precision@1": 0.54, + "precision@5": 0.15599999999999997, + "precision@10": 0.08599999999999998, + "recall@1": 0.54, + "recall@5": 0.78, + "recall@10": 0.86, + "f1@1": 0.54, + "f1@5": 0.26, + "f1@10": 0.15636363636363637, + "map@1": 0.54, + "map@5": 0.6251666666666666, + "map@10": 0.6354523809523809, + "mrr@1": 0.54, + "mrr@5": 0.6251666666666666, + "mrr@10": 0.6354523809523809, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_government_reports": { "hit@1": 0.58, - "hit@5": 0.84, + "hit@5": 0.85, "hit@10": 0.88, "ndcg_linear@1": 0.58, - "ndcg_linear@5": 0.7215467489235274, - "ndcg_linear@10": 0.7346071028524384, + "ndcg_linear@5": 0.7221034475051777, + "ndcg_linear@10": 0.7312786961863147, "ndcg_exponential@1": 0.58, - "ndcg_exponential@5": 0.7215467489235274, - "ndcg_exponential@10": 0.7346071028524384, + "ndcg_exponential@5": 0.7221034475051777, + "ndcg_exponential@10": 0.7312786961863147, "precision@1": 0.58, - "precision@5": 0.16799999999999998, + "precision@5": 0.16999999999999996, "precision@10": 0.088, "recall@1": 0.58, - "recall@5": 0.84, + "recall@5": 0.85, "recall@10": 0.88, "f1@1": 0.58, - "f1@5": 0.2800000000000001, + "f1@5": 0.2833333333333334, "f1@10": 0.16000000000000003, "map@1": 0.58, - "map@5": 0.6818333333333332, - "map@10": 0.6872896825396825, + "map@5": 0.6796666666666665, + "map@10": 0.6831388888888889, "mrr@1": 0.58, - "mrr@5": 0.6818333333333332, - "mrr@10": 0.6872896825396825, + "mrr@5": 0.6796666666666665, + "mrr@10": 0.6831388888888889, "num_pred": 100, "num_data": 100 }, @@ -1781,11 +1809,11 @@ "hit@5": 0.94, "hit@10": 0.98, "ndcg_linear@1": 0.7, - "ndcg_linear@5": 0.8374771734856519, - "ndcg_linear@10": 0.8501891371833549, + "ndcg_linear@5": 0.837992582296858, + "ndcg_linear@10": 0.8507045459945608, "ndcg_exponential@1": 0.7, - "ndcg_exponential@5": 0.8374771734856519, - "ndcg_exponential@10": 0.8501891371833549, + "ndcg_exponential@5": 0.837992582296858, + "ndcg_exponential@10": 0.8507045459945608, "precision@1": 0.7, "precision@5": 0.18799999999999997, "precision@10": 0.09799999999999998, @@ -1796,403 +1824,347 @@ "f1@5": 0.3133333333333334, "f1@10": 0.17818181818181827, "map@1": 0.7, - "map@5": 0.8026666666666666, - "map@10": 0.8077738095238094, + "map@5": 0.8031666666666666, + "map@10": 0.8082738095238093, "mrr@1": 0.7, - "mrr@5": 0.8026666666666666, - "mrr@10": 0.8077738095238094, + "mrr@5": 0.8031666666666666, + "mrr@10": 0.8082738095238093, "num_pred": 100, "num_data": 100 }, "ViDoRe_esg_reports_human_labeled_v2": { - "hit@1": 0.28846153846153844, - "hit@5": 0.4807692307692308, - "hit@10": 0.6153846153846154, - "ndcg_linear@1": 0.28846153846153844, - "ndcg_linear@5": 0.3295140713668188, - "ndcg_linear@10": 0.3803485934947541, - "ndcg_exponential@1": 0.28846153846153844, - "ndcg_exponential@5": 0.3296697471059318, - "ndcg_exponential@10": 0.3804833955905218, - "precision@1": 0.28846153846153844, - "precision@5": 0.12692307692307694, - "precision@10": 0.09423076923076924, - "recall@1": 0.20576923076923076, - "recall@5": 0.3809149184149184, - "recall@10": 0.5121503496503498, - "f1@1": 0.2269230769230769, - "f1@5": 0.17453449328449328, - "f1@10": 0.14645786905402286, - "map@1": 0.28846153846153844, - "map@5": 0.2831730769230769, - "map@10": 0.30995421245421245, - "mrr@1": 0.28846153846153844, - "mrr@5": 0.36025641025641025, - "mrr@10": 0.378968253968254, + "hit@1": 0.3076923076923077, + "hit@5": 0.5, + "hit@10": 0.6346153846153846, + "ndcg_linear@1": 0.3076923076923077, + "ndcg_linear@5": 0.32983882678962817, + "ndcg_linear@10": 0.38161393426101153, + "ndcg_exponential@1": 0.3076923076923077, + "ndcg_exponential@5": 0.33007080785107673, + "ndcg_exponential@10": 0.38204774409431586, + "precision@1": 0.3076923076923077, + "precision@5": 0.13076923076923078, + "precision@10": 0.09423076923076922, + "recall@1": 0.20961538461538462, + "recall@5": 0.3799533799533799, + "recall@10": 0.5265734265734267, + "f1@1": 0.23333333333333334, + "f1@5": 0.17583539826186886, + "f1@10": 0.14720711980327364, + "map@1": 0.3076923076923077, + "map@5": 0.28035256410256415, + "map@10": 0.3043701668701669, + "mrr@1": 0.3076923076923077, + "mrr@5": 0.3721153846153846, + "mrr@10": 0.3888278388278389, "num_pred": 52, "num_data": 52 }, "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.2859375, - "hit@5": 0.5859375, - "hit@10": 0.684375, - "ndcg_linear@1": 0.2859375, - "ndcg_linear@5": 0.35874452751062924, - "ndcg_linear@10": 0.39816762044104737, - "ndcg_exponential@1": 0.2859375, - "ndcg_exponential@5": 0.35874452751062924, - "ndcg_exponential@10": 0.39816762044104737, - "precision@1": 0.2859375, - "precision@5": 0.1653125, - "precision@10": 0.11640625, - "recall@1": 0.1754330307750059, - "recall@5": 0.4161406827621331, - "recall@10": 0.5186508895780099, - "f1@1": 0.19900094696969695, - "f1@5": 0.2121714847142237, - "f1@10": 0.17188047002017592, - "map@1": 0.2859375, - "map@5": 0.2968263888888889, - "map@10": 0.3171867721915155, - "mrr@1": 0.2859375, - "mrr@5": 0.4043489583333333, - "mrr@10": 0.4175502232142857, + "hit@1": 0.284375, + "hit@5": 0.578125, + "hit@10": 0.6875, + "ndcg_linear@1": 0.284375, + "ndcg_linear@5": 0.358673242491213, + "ndcg_linear@10": 0.3980581354160106, + "ndcg_exponential@1": 0.284375, + "ndcg_exponential@5": 0.358673242491213, + "ndcg_exponential@10": 0.3980581354160106, + "precision@1": 0.284375, + "precision@5": 0.16499999999999998, + "precision@10": 0.11546875000000001, + "recall@1": 0.17665451672440136, + "recall@5": 0.4158029070268389, + "recall@10": 0.5191667294366805, + "f1@1": 0.1997859172077922, + "f1@5": 0.21200744389393292, + "f1@10": 0.1708723781664958, + "map@1": 0.284375, + "map@5": 0.29842838541666666, + "map@10": 0.31832021875590516, + "mrr@1": 0.284375, + "mrr@5": 0.40015625000000005, + "mrr@10": 0.4145300099206349, "num_pred": 640, "num_data": 640 }, "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.3706896551724138, - "hit@5": 0.75, - "hit@10": 0.8189655172413793, - "ndcg_linear@1": 0.3706896551724138, - "ndcg_linear@5": 0.3194451317176268, - "ndcg_linear@10": 0.3202641880538733, - "ndcg_exponential@1": 0.3706896551724138, - "ndcg_exponential@5": 0.3194451317176268, - "ndcg_exponential@10": 0.3202641880538733, - "precision@1": 0.3706896551724138, - "precision@5": 0.2896551724137932, - "precision@10": 0.24870689655172415, - "recall@1": 0.03865338906185176, - "recall@5": 0.17160160948032535, - "recall@10": 0.25447962863698914, - "f1@1": 0.06726379557208981, - "f1@5": 0.1779005506820022, - "f1@10": 0.20701322326238306, - "map@1": 0.3706896551724138, - "map@5": 0.21762931034482758, - "map@10": 0.19331952955990825, - "mrr@1": 0.3706896551724138, - "mrr@5": 0.5000718390804598, - "mrr@10": 0.5099839217296114, + "hit@1": 0.38362068965517243, + "hit@5": 0.7543103448275862, + "hit@10": 0.8318965517241379, + "ndcg_linear@1": 0.38362068965517243, + "ndcg_linear@5": 0.32368274417981013, + "ndcg_linear@10": 0.31983898076847644, + "ndcg_exponential@1": 0.38362068965517243, + "ndcg_exponential@5": 0.32368274417981013, + "ndcg_exponential@10": 0.31983898076847644, + "precision@1": 0.38362068965517243, + "precision@5": 0.2948275862068966, + "precision@10": 0.24655172413793106, + "recall@1": 0.03750144190924211, + "recall@5": 0.1693184160726054, + "recall@10": 0.2565797980641253, + "f1@1": 0.0662670827542563, + "f1@5": 0.17931590136463726, + "f1@10": 0.20636592569893428, + "map@1": 0.38362068965517243, + "map@5": 0.22136494252873565, + "map@10": 0.19123329999304955, + "mrr@1": 0.38362068965517243, + "mrr@5": 0.5096264367816092, + "mrr@10": 0.520734126984127, "num_pred": 232, "num_data": 232 }, "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.23684210526315788, + "hit@1": 0.2543859649122807, "hit@5": 0.6578947368421053, - "hit@10": 0.7587719298245614, - "ndcg_linear@1": 0.23684210526315788, - "ndcg_linear@5": 0.32514161979506145, - "ndcg_linear@10": 0.3716042928744199, - "ndcg_exponential@1": 0.23684210526315788, - "ndcg_exponential@5": 0.32514161979506145, - "ndcg_exponential@10": 0.3716042928744199, - "precision@1": 0.23684210526315788, - "precision@5": 0.17982456140350878, + "hit@10": 0.7631578947368421, + "ndcg_linear@1": 0.2543859649122807, + "ndcg_linear@5": 0.3325835928626141, + "ndcg_linear@10": 0.38047437753455154, + "ndcg_exponential@1": 0.2543859649122807, + "ndcg_exponential@5": 0.3325835928626141, + "ndcg_exponential@10": 0.38047437753455154, + "precision@1": 0.2543859649122807, + "precision@5": 0.17894736842105263, "precision@10": 0.13552631578947372, - "recall@1": 0.10328947368421053, - "recall@5": 0.3982769423558897, - "recall@10": 0.5284043441938179, - "f1@1": 0.1284477705530337, - "f1@5": 0.21337619006696623, - "f1@10": 0.18932479489653187, - "map@1": 0.23684210526315788, - "map@5": 0.2391544834307992, - "map@10": 0.25983787593984964, - "mrr@1": 0.23684210526315788, - "mrr@5": 0.38559941520467833, - "mrr@10": 0.39949526594263435, + "recall@1": 0.11681286549707602, + "recall@5": 0.39830304928989135, + "recall@10": 0.5307435254803676, + "f1@1": 0.14385487806540434, + "f1@5": 0.21427602478499247, + "f1@10": 0.18972364498578448, + "map@1": 0.2543859649122807, + "map@5": 0.24812621832358672, + "map@10": 0.27017407523438225, + "mrr@1": 0.2543859649122807, + "mrr@5": 0.40131578947368424, + "mrr@10": 0.41581558061821217, "num_pred": 228, "num_data": 228 }, "VisRAG_ArxivQA": { - "hit@1": 0.29901960784313725, - "hit@5": 0.44362745098039214, - "hit@10": 0.508578431372549, - "ndcg_linear@1": 0.29901960784313725, - "ndcg_linear@5": 0.37679708973810255, - "ndcg_linear@10": 0.3979379930260254, - "ndcg_exponential@1": 0.29901960784313725, - "ndcg_exponential@5": 0.37679708973810255, - "ndcg_exponential@10": 0.3979379930260254, - "precision@1": 0.29901960784313725, - "precision@5": 0.08872549019607844, - "precision@10": 0.0508578431372549, - "recall@1": 0.29901960784313725, - "recall@5": 0.44362745098039214, - "recall@10": 0.508578431372549, - "f1@1": 0.29901960784313725, - "f1@5": 0.14787581699346408, - "f1@10": 0.09246880570409984, - "map@1": 0.29901960784313725, - "map@5": 0.35451388888888885, - "map@10": 0.36331602085278564, - "mrr@1": 0.29901960784313725, - "mrr@5": 0.35451388888888885, - "mrr@10": 0.36331602085278564, + "hit@1": 0.30637254901960786, + "hit@5": 0.44485294117647056, + "hit@10": 0.5110294117647058, + "ndcg_linear@1": 0.30637254901960786, + "ndcg_linear@5": 0.380316616602221, + "ndcg_linear@10": 0.40192177212626184, + "ndcg_exponential@1": 0.30637254901960786, + "ndcg_exponential@5": 0.380316616602221, + "ndcg_exponential@10": 0.40192177212626184, + "precision@1": 0.30637254901960786, + "precision@5": 0.0889705882352941, + "precision@10": 0.05110294117647058, + "recall@1": 0.30637254901960786, + "recall@5": 0.44485294117647056, + "recall@10": 0.5110294117647058, + "f1@1": 0.30637254901960786, + "f1@5": 0.14828431372549022, + "f1@10": 0.0929144385026738, + "map@1": 0.30637254901960786, + "map@5": 0.35880310457516335, + "map@10": 0.3678449852163088, + "mrr@1": 0.30637254901960786, + "mrr@5": 0.35880310457516335, + "mrr@10": 0.3678449852163088, "num_pred": 816, "num_data": 816 }, "VisRAG_ChartQA": { - "hit@1": 0.5873015873015873, + "hit@1": 0.5714285714285714, "hit@5": 0.7142857142857143, "hit@10": 0.746031746031746, - "ndcg_linear@1": 0.5873015873015873, - "ndcg_linear@5": 0.6589841568889536, - "ndcg_linear@10": 0.6681608180419024, - "ndcg_exponential@1": 0.5873015873015873, - "ndcg_exponential@5": 0.6589841568889536, - "ndcg_exponential@10": 0.6681608180419024, - "precision@1": 0.5873015873015873, + "ndcg_linear@1": 0.5714285714285714, + "ndcg_linear@5": 0.6521480210465188, + "ndcg_linear@10": 0.6615146055224214, + "ndcg_exponential@1": 0.5714285714285714, + "ndcg_exponential@5": 0.6521480210465188, + "ndcg_exponential@10": 0.6615146055224214, + "precision@1": 0.5714285714285714, "precision@5": 0.14285714285714282, "precision@10": 0.07460317460317457, - "recall@1": 0.5873015873015873, + "recall@1": 0.5714285714285714, "recall@5": 0.7142857142857143, "recall@10": 0.746031746031746, - "f1@1": 0.5873015873015873, + "f1@1": 0.5714285714285714, "f1@5": 0.23809523809523817, "f1@10": 0.13564213564213565, - "map@1": 0.5873015873015873, - "map@5": 0.6402116402116402, - "map@10": 0.6433862433862434, - "mrr@1": 0.5873015873015873, - "mrr@5": 0.6402116402116402, - "mrr@10": 0.6433862433862434, + "map@1": 0.5714285714285714, + "map@5": 0.630952380952381, + "map@10": 0.6343033509700177, + "mrr@1": 0.5714285714285714, + "mrr@5": 0.630952380952381, + "mrr@10": 0.6343033509700177, "num_pred": 63, "num_data": 63 }, "VisRAG_MP-DocVQA": { - "hit@1": 0.40439932318104904, - "hit@5": 0.6700507614213198, - "hit@10": 0.7377326565143824, - "ndcg_linear@1": 0.40439932318104904, - "ndcg_linear@5": 0.5452225541556516, - "ndcg_linear@10": 0.5673639059314857, - "ndcg_exponential@1": 0.40439932318104904, - "ndcg_exponential@5": 0.5452225541556516, - "ndcg_exponential@10": 0.5673639059314857, - "precision@1": 0.40439932318104904, - "precision@5": 0.13401015228426394, - "precision@10": 0.07377326565143823, - "recall@1": 0.40439932318104904, - "recall@5": 0.6700507614213198, - "recall@10": 0.7377326565143824, - "f1@1": 0.40439932318104904, - "f1@5": 0.22335025380710666, - "f1@10": 0.13413321027534228, - "map@1": 0.40439932318104904, - "map@5": 0.5038071065989848, - "map@10": 0.5130885236215186, - "mrr@1": 0.40439932318104904, - "mrr@5": 0.5038071065989848, - "mrr@10": 0.5130885236215186, + "hit@1": 0.4077834179357022, + "hit@5": 0.6615905245346869, + "hit@10": 0.7461928934010152, + "ndcg_linear@1": 0.4077834179357022, + "ndcg_linear@5": 0.5413871684319078, + "ndcg_linear@10": 0.5686420676375945, + "ndcg_exponential@1": 0.4077834179357022, + "ndcg_exponential@5": 0.5413871684319078, + "ndcg_exponential@10": 0.5686420676375945, + "precision@1": 0.4077834179357022, + "precision@5": 0.1323181049069374, + "precision@10": 0.07461928934010151, + "recall@1": 0.4077834179357022, + "recall@5": 0.6615905245346869, + "recall@10": 0.7461928934010152, + "f1@1": 0.4077834179357022, + "f1@5": 0.22053017484489573, + "f1@10": 0.13567143516382096, + "map@1": 0.4077834179357022, + "map@5": 0.501551043429216, + "map@10": 0.5127353423038702, + "mrr@1": 0.4077834179357022, + "mrr@5": 0.501551043429216, + "mrr@10": 0.5127353423038702, "num_pred": 591, "num_data": 591 }, "VisRAG_SlideVQA": { - "hit@1": 0.6690647482014388, + "hit@1": 0.6726618705035972, "hit@5": 0.8830935251798561, - "hit@10": 0.9190647482014388, - "ndcg_linear@1": 0.6690647482014388, - "ndcg_linear@5": 0.765424898996163, - "ndcg_linear@10": 0.7831852550058264, - "ndcg_exponential@1": 0.6690647482014388, - "ndcg_exponential@5": 0.765424898996163, - "ndcg_exponential@10": 0.7831852550058264, - "precision@1": 0.6690647482014388, - "precision@5": 0.21546762589928053, - "precision@10": 0.11438848920863308, - "recall@1": 0.5701438848920863, - "recall@5": 0.8552158273381295, - "recall@10": 0.9055755395683454, - "f1@1": 0.6031175059952038, - "f1@5": 0.33795820486467976, - "f1@10": 0.2007575757575758, - "map@1": 0.6690647482014388, - "map@5": 0.7239658273381294, - "map@10": 0.7330735411670664, - "mrr@1": 0.6690647482014388, - "mrr@5": 0.7542865707434052, - "mrr@10": 0.7592818602261048, + "hit@10": 0.920863309352518, + "ndcg_linear@1": 0.6726618705035972, + "ndcg_linear@5": 0.7655156212899079, + "ndcg_linear@10": 0.7839387351442914, + "ndcg_exponential@1": 0.6726618705035972, + "ndcg_exponential@5": 0.7655156212899079, + "ndcg_exponential@10": 0.7839387351442914, + "precision@1": 0.6726618705035972, + "precision@5": 0.2158273381294964, + "precision@10": 0.11474820143884891, + "recall@1": 0.5719424460431655, + "recall@5": 0.8561151079136691, + "recall@10": 0.908273381294964, + "f1@1": 0.6055155875299759, + "f1@5": 0.33847207947927377, + "f1@10": 0.20138434706780037, + "map@1": 0.6726618705035972, + "map@5": 0.7235611510791367, + "map@10": 0.7330132322713258, + "mrr@1": 0.6726618705035972, + "mrr@5": 0.7548860911270983, + "mrr@10": 0.7601704350805071, "num_pred": 556, "num_data": 556 }, "VisRAG_InfoVQA": { - "hit@1": 0.5974930362116991, - "hit@5": 0.8454038997214485, - "hit@10": 0.8955431754874652, - "ndcg_linear@1": 0.5974930362116991, - "ndcg_linear@5": 0.7325704078178448, - "ndcg_linear@10": 0.749399435766671, - "ndcg_exponential@1": 0.5974930362116991, - "ndcg_exponential@5": 0.7325704078178448, - "ndcg_exponential@10": 0.749399435766671, - "precision@1": 0.5974930362116991, - "precision@5": 0.16908077994428966, - "precision@10": 0.08955431754874651, - "recall@1": 0.5974930362116991, - "recall@5": 0.8454038997214485, - "recall@10": 0.8955431754874652, - "f1@1": 0.5974930362116991, - "f1@5": 0.2818012999071495, - "f1@10": 0.16282603190681189, - "map@1": 0.5974930362116991, - "map@5": 0.6947075208913649, - "map@10": 0.7020200512888535, - "mrr@1": 0.5974930362116991, - "mrr@5": 0.6947075208913649, - "mrr@10": 0.7020200512888535, + "hit@1": 0.6002785515320335, + "hit@5": 0.8481894150417827, + "hit@10": 0.8997214484679665, + "ndcg_linear@1": 0.6002785515320335, + "ndcg_linear@5": 0.7350159831014981, + "ndcg_linear@10": 0.7519780130085987, + "ndcg_exponential@1": 0.6002785515320335, + "ndcg_exponential@5": 0.7350159831014981, + "ndcg_exponential@10": 0.7519780130085987, + "precision@1": 0.6002785515320335, + "precision@5": 0.16963788300835653, + "precision@10": 0.08997214484679665, + "recall@1": 0.6002785515320335, + "recall@5": 0.8481894150417827, + "recall@10": 0.8997214484679665, + "f1@1": 0.6002785515320335, + "f1@5": 0.28272980501392764, + "f1@10": 0.16358571790326668, + "map@1": 0.6002785515320335, + "map@5": 0.697075208913649, + "map@10": 0.7042567537692885, + "mrr@1": 0.6002785515320335, + "mrr@5": 0.697075208913649, + "mrr@10": 0.7042567537692885, "num_pred": 718, "num_data": 718 }, "VisRAG_PlotQA": { - "hit@1": 0.28389339513325607, - "hit@5": 0.526071842410197, - "hit@10": 0.6303592120509849, - "ndcg_linear@1": 0.28389339513325607, - "ndcg_linear@5": 0.4120322200394941, - "ndcg_linear@10": 0.4459578429761404, - "ndcg_exponential@1": 0.28389339513325607, - "ndcg_exponential@5": 0.4120322200394941, - "ndcg_exponential@10": 0.4459578429761404, - "precision@1": 0.28389339513325607, - "precision@5": 0.10521436848203938, - "precision@10": 0.06303592120509849, - "recall@1": 0.28389339513325607, - "recall@5": 0.526071842410197, - "recall@10": 0.6303592120509849, - "f1@1": 0.28389339513325607, - "f1@5": 0.17535728080339905, - "f1@10": 0.11461076582745183, - "map@1": 0.28389339513325607, - "map@5": 0.37414059482425643, - "map@10": 0.38825709503577405, - "mrr@1": 0.28389339513325607, - "mrr@5": 0.37414059482425643, - "mrr@10": 0.38825709503577405, + "hit@1": 0.2954808806488992, + "hit@5": 0.522595596755504, + "hit@10": 0.6280417149478563, + "ndcg_linear@1": 0.2954808806488992, + "ndcg_linear@5": 0.4146310437293564, + "ndcg_linear@10": 0.4491487424642551, + "ndcg_exponential@1": 0.2954808806488992, + "ndcg_exponential@5": 0.4146310437293564, + "ndcg_exponential@10": 0.4491487424642551, + "precision@1": 0.2954808806488992, + "precision@5": 0.1045191193511008, + "precision@10": 0.06280417149478562, + "recall@1": 0.2954808806488992, + "recall@5": 0.522595596755504, + "recall@10": 0.6280417149478563, + "f1@1": 0.2954808806488992, + "f1@5": 0.17419853225183474, + "f1@10": 0.11418940271779207, + "map@1": 0.2954808806488992, + "map@5": 0.3788142139822325, + "map@10": 0.39330500836873955, + "mrr@1": 0.2954808806488992, + "mrr@5": 0.3788142139822325, + "mrr@10": 0.39330500836873955, "num_pred": 863, "num_data": 863 }, - "ViDoSeek-page": { - "hit@1": 0.0446584938704028, - "hit@5": 0.4063047285464098, - "hit@10": 0.6007005253940455, - "ndcg_linear@1": 0.0446584938704028, - "ndcg_linear@5": 0.23051188319719665, - "ndcg_linear@10": 0.29430171980440295, - "ndcg_exponential@1": 0.0446584938704028, - "ndcg_exponential@5": 0.2305118831971967, - "ndcg_exponential@10": 0.29430171980440295, - "precision@1": 0.0446584938704028, - "precision@5": 0.08126094570928197, - "precision@10": 0.06007005253940455, - "recall@1": 0.0446584938704028, - "recall@5": 0.4063047285464098, - "recall@10": 0.6007005253940455, - "f1@1": 0.0446584938704028, - "f1@5": 0.13543490951546996, - "f1@10": 0.10921827734437194, - "map@1": 0.0446584938704028, - "map@5": 0.17269410391126677, - "map@10": 0.19956008673171546, - "mrr@1": 0.0446584938704028, - "mrr@5": 0.17269410391126677, - "mrr@10": 0.19956008673171546, + "ViDoSeek-page-fixed": { + "hit@1": 0.5122591943957968, + "hit@5": 0.8336252189141856, + "hit@10": 0.9185639229422067, + "ndcg_linear@1": 0.5122591943957968, + "ndcg_linear@5": 0.685369553267636, + "ndcg_linear@10": 0.712893633896179, + "ndcg_exponential@1": 0.5122591943957968, + "ndcg_exponential@5": 0.685369553267636, + "ndcg_exponential@10": 0.712893633896179, + "precision@1": 0.5122591943957968, + "precision@5": 0.16672504378283712, + "precision@10": 0.09185639229422064, + "recall@1": 0.5122591943957968, + "recall@5": 0.8336252189141856, + "recall@10": 0.9185639229422067, + "f1@1": 0.5122591943957968, + "f1@5": 0.2778750729713953, + "f1@10": 0.1670116223531285, + "map@1": 0.5122591943957968, + "map@5": 0.6359019264448337, + "map@10": 0.6472934562032636, + "mrr@1": 0.5122591943957968, + "mrr@5": 0.6359019264448337, + "mrr@10": 0.6472934562032636, "num_pred": 1142, "num_data": 1142 }, - "ViDoSeek-doc": { - "hit@1": 0.9658493870402802, - "hit@5": 0.9903677758318739, - "hit@10": 0.9912434325744308, - "ndcg_linear@1": 0.6614127262113251, - "ndcg_linear@5": 0.8026274011129774, - "ndcg_linear@10": 0.793858821519423, - "ndcg_exponential@1": 0.4439579684763572, - "ndcg_exponential@5": 0.6878057015219627, - "ndcg_exponential@10": 0.7216552842804831, - "precision@1": 0.9658493870402802, - "precision@5": 0.8849387040280211, - "precision@10": 0.8094570928196146, - "recall@1": 0.056997106757459524, - "recall@5": 0.25965311344637504, - "recall@10": 0.4648221767265056, - "f1@1": 0.10690967591559217, - "f1@5": 0.39332503261123863, - "f1@10": 0.5772393543236635, - "map@1": 0.9658493870402802, - "map@5": 0.8711587857559837, - "map@10": 0.7917361333114032, - "mrr@1": 0.9658493870402802, - "mrr@5": 0.9748540572095739, - "mrr@10": 0.974979151029939, - "num_pred": 1142, - "num_data": 1142 - }, - "MMLongBench-page": { - "hit@1": 0.07040572792362769, - "hit@5": 0.26968973747016706, - "hit@10": 0.3711217183770883, - "ndcg_linear@1": 0.07040572792362769, - "ndcg_linear@5": 0.1352234900192414, - "ndcg_linear@10": 0.17032905812486968, - "ndcg_exponential@1": 0.07040572792362769, - "ndcg_exponential@5": 0.1352234900192414, - "ndcg_exponential@10": 0.1703290581248697, - "precision@1": 0.07040572792362769, - "precision@5": 0.0613365155131265, - "precision@10": 0.046420047732696895, - "recall@1": 0.042114160700079555, - "recall@5": 0.19418257756563248, - "recall@10": 0.2923252860902026, - "f1@1": 0.05010710583502946, - "f1@5": 0.08893912800974434, - "f1@10": 0.07756931033754735, - "map@1": 0.07040572792362769, - "map@5": 0.10046904004242907, - "map@10": 0.11652881167051811, - "mrr@1": 0.07040572792362769, - "mrr@5": 0.14013524264120925, - "mrr@10": 0.15354159563586772, - "num_pred": 838, - "num_data": 838 - }, - "MMLongBench-doc": { - "hit@1": 0.548926014319809, - "hit@5": 0.6825775656324582, - "hit@10": 0.7303102625298329, - "ndcg_linear@1": 0.386634844868735, - "ndcg_linear@5": 0.43544677369246687, - "ndcg_linear@10": 0.4315957106632966, - "ndcg_exponential@1": 0.2707125809751108, - "ndcg_exponential@5": 0.36475310712288217, - "ndcg_exponential@10": 0.3840612564382075, - "precision@1": 0.548926014319809, - "precision@5": 0.4911694510739857, - "precision@10": 0.45083532219570405, - "recall@1": 0.016616151041636364, - "recall@5": 0.0724119472425789, - "recall@10": 0.1306605325823178, - "f1@1": 0.0319233655487925, - "f1@5": 0.12112180979310318, - "f1@10": 0.19000676907704542, - "map@1": 0.548926014319809, - "map@5": 0.45691328560063643, - "map@10": 0.40530936280637947, - "mrr@1": 0.548926014319809, - "mrr@5": 0.5991845664280032, - "mrr@10": 0.6056853051483123, + "MMLongBench-page-fixed": { + "hit@1": 0.29952267303102625, + "hit@5": 0.4916467780429594, + "hit@10": 0.5775656324582339, + "ndcg_linear@1": 0.29952267303102625, + "ndcg_linear@5": 0.3582863524406764, + "ndcg_linear@10": 0.3928597838193395, + "ndcg_exponential@1": 0.29952267303102625, + "ndcg_exponential@5": 0.3582863524406764, + "ndcg_exponential@10": 0.3928597838193395, + "precision@1": 0.29952267303102625, + "precision@5": 0.120763723150358, + "precision@10": 0.07649164677804295, + "recall@1": 0.22916666666666666, + "recall@5": 0.4214299920445505, + "recall@10": 0.5187089660538348, + "f1@1": 0.2507208119379957, + "f1@5": 0.1807137108449758, + "f1@10": 0.1289270983224814, + "map@1": 0.29952267303102625, + "map@5": 0.31855045080880395, + "map@10": 0.33536877217739386, + "mrr@1": 0.29952267303102625, + "mrr@5": 0.3700676213206046, + "mrr@10": 0.3811825207409933, "num_pred": 838, "num_data": 838 }