diff --git "a/scores/VLM2Vec-V2.0-Qwen2VL-2B.json" "b/scores/VLM2Vec-V2.0-Qwen2VL-2B.json" --- "a/scores/VLM2Vec-V2.0-Qwen2VL-2B.json" +++ "b/scores/VLM2Vec-V2.0-Qwen2VL-2B.json" @@ -1,7 +1,7 @@ { "metadata": { "model_name": "VLM2Vec-V2.0-Qwen2VL-2B", - "report_generated_date": "2025-06-28T08:54:17.882723", + "report_generated_date": "2025-12-29T09:27:50.301732", "model_size": 2.21, "url": "https://huggingface.co/VLM2Vec/VLM2Vec-V2.0", "data_source": "TIGER-Lab" @@ -13,11 +13,11 @@ "hit@5": 0.956, "hit@10": 0.981, "ndcg_linear@1": 0.808, - "ndcg_linear@5": 0.8917355668881671, - "ndcg_linear@10": 0.8996344961395271, + "ndcg_linear@5": 0.8918587795114569, + "ndcg_linear@10": 0.899731308712349, "ndcg_exponential@1": 0.808, - "ndcg_exponential@5": 0.8917355668881671, - "ndcg_exponential@10": 0.8996344961395271, + "ndcg_exponential@5": 0.8918587795114569, + "ndcg_exponential@10": 0.899731308712349, "precision@1": 0.808, "precision@5": 0.19120000000000004, "precision@10": 0.09810000000000002, @@ -28,1473 +28,1501 @@ "f1@5": 0.3186666666666667, "f1@10": 0.17836363636363634, "map@1": 0.808, - "map@5": 0.8699166666666668, - "map@10": 0.8730702380952381, + "map@5": 0.8700833333333333, + "map@10": 0.8732119047619049, "mrr@1": 0.808, - "mrr@5": 0.8699166666666668, - "mrr@10": 0.8730702380952381, + "mrr@5": 0.8700833333333333, + "mrr@10": 0.8732119047619049, "num_pred": 1000, "num_data": 1000 }, "N24News": { - "hit@1": 0.729, - "hit@5": 0.954, + "hit@1": 0.73, + "hit@5": 0.956, "hit@10": 0.985, - "ndcg_linear@1": 0.729, - "ndcg_linear@5": 0.8542486272909989, - "ndcg_linear@10": 0.8643817096780151, - "ndcg_exponential@1": 0.729, - "ndcg_exponential@5": 0.8542486272909989, - "ndcg_exponential@10": 0.8643817096780151, - "precision@1": 0.729, - "precision@5": 0.19080000000000003, + "ndcg_linear@1": 0.73, + "ndcg_linear@5": 0.8551983255877351, + "ndcg_linear@10": 0.864717448787753, + "ndcg_exponential@1": 0.73, + "ndcg_exponential@5": 0.8551983255877351, + "ndcg_exponential@10": 0.864717448787753, + "precision@1": 0.73, + "precision@5": 0.1912, "precision@10": 0.09850000000000003, - "recall@1": 0.729, - "recall@5": 0.954, + "recall@1": 0.73, + "recall@5": 0.956, "recall@10": 0.985, - "f1@1": 0.729, - "f1@5": 0.318, + "f1@1": 0.73, + "f1@5": 0.3186666666666667, "f1@10": 0.17909090909090908, - "map@1": 0.729, - "map@5": 0.82055, - "map@10": 0.8247988095238095, - "mrr@1": 0.729, - "mrr@5": 0.82055, - "mrr@10": 0.8247988095238095, + "map@1": 0.73, + "map@5": 0.8212166666666667, + "map@10": 0.8252325396825397, + "mrr@1": 0.73, + "mrr@5": 0.8212166666666667, + "mrr@10": 0.8252325396825397, "num_pred": 1000, "num_data": 1000 }, "HatefulMemes": { - "hit@1": 0.563, + "hit@1": 0.558, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.563, - "ndcg_linear@5": 0.8387163023107271, - "ndcg_linear@10": 0.8387163023107271, - "ndcg_exponential@1": 0.563, - "ndcg_exponential@5": 0.8387163023107271, - "ndcg_exponential@10": 0.8387163023107271, - "precision@1": 0.563, + "ndcg_linear@1": 0.558, + "ndcg_linear@5": 0.8368709510785843, + "ndcg_linear@10": 0.8368709510785843, + "ndcg_exponential@1": 0.558, + "ndcg_exponential@5": 0.8368709510785843, + "ndcg_exponential@10": 0.8368709510785843, + "precision@1": 0.558, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.563, + "recall@1": 0.558, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.563, + "f1@1": 0.558, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.563, - "map@5": 0.7815, - "map@10": 0.7815, - "mrr@1": 0.563, - "mrr@5": 0.7815, - "mrr@10": 0.7815, + "map@1": 0.558, + "map@5": 0.779, + "map@10": 0.779, + "mrr@1": 0.558, + "mrr@5": 0.779, + "mrr@10": 0.779, "num_pred": 1000, "num_data": 1000 }, "VOC2007": { - "hit@1": 0.85, + "hit@1": 0.849, "hit@5": 0.975, "hit@10": 0.988, - "ndcg_linear@1": 0.85, - "ndcg_linear@5": 0.920705885295906, - "ndcg_linear@10": 0.9248925595376308, - "ndcg_exponential@1": 0.85, - "ndcg_exponential@5": 0.920705885295906, - "ndcg_exponential@10": 0.9248925595376308, - "precision@1": 0.85, + "ndcg_linear@1": 0.849, + "ndcg_linear@5": 0.9207811451913126, + "ndcg_linear@10": 0.9250408650810289, + "ndcg_exponential@1": 0.849, + "ndcg_exponential@5": 0.9207811451913126, + "ndcg_exponential@10": 0.9250408650810289, + "precision@1": 0.849, "precision@5": 0.19500000000000006, "precision@10": 0.09880000000000001, - "recall@1": 0.85, + "recall@1": 0.849, "recall@5": 0.975, "recall@10": 0.988, - "f1@1": 0.85, + "f1@1": 0.849, "f1@5": 0.325, "f1@10": 0.17963636363636362, - "map@1": 0.85, - "map@5": 0.9022666666666667, - "map@10": 0.9039853174603174, - "mrr@1": 0.85, - "mrr@5": 0.9022666666666667, - "mrr@10": 0.9039853174603174, + "map@1": 0.849, + "map@5": 0.9023166666666667, + "map@10": 0.9041087301587302, + "mrr@1": 0.849, + "mrr@5": 0.9023166666666667, + "mrr@10": 0.9041087301587302, "num_pred": 1000, "num_data": 1000 }, "SUN397": { - "hit@1": 0.71, - "hit@5": 0.93, - "hit@10": 0.957, - "ndcg_linear@1": 0.71, - "ndcg_linear@5": 0.8327114376725517, - "ndcg_linear@10": 0.841469031189054, - "ndcg_exponential@1": 0.71, - "ndcg_exponential@5": 0.8327114376725517, - "ndcg_exponential@10": 0.841469031189054, - "precision@1": 0.71, - "precision@5": 0.186, - "precision@10": 0.09570000000000002, - "recall@1": 0.71, - "recall@5": 0.93, - "recall@10": 0.957, - "f1@1": 0.71, - "f1@5": 0.31, - "f1@10": 0.174, - "map@1": 0.71, - "map@5": 0.7998666666666666, - "map@10": 0.803498015873016, - "mrr@1": 0.71, - "mrr@5": 0.7998666666666666, - "mrr@10": 0.803498015873016, + "hit@1": 0.709, + "hit@5": 0.931, + "hit@10": 0.956, + "ndcg_linear@1": 0.709, + "ndcg_linear@5": 0.8329377320580826, + "ndcg_linear@10": 0.8409819473367187, + "ndcg_exponential@1": 0.709, + "ndcg_exponential@5": 0.8329377320580826, + "ndcg_exponential@10": 0.8409819473367187, + "precision@1": 0.709, + "precision@5": 0.1862, + "precision@10": 0.09560000000000002, + "recall@1": 0.709, + "recall@5": 0.931, + "recall@10": 0.956, + "f1@1": 0.709, + "f1@5": 0.3103333333333333, + "f1@10": 0.17381818181818182, + "map@1": 0.709, + "map@5": 0.7998, + "map@10": 0.8030968253968254, + "mrr@1": 0.709, + "mrr@5": 0.7998, + "mrr@10": 0.8030968253968254, "num_pred": 1000, "num_data": 1000 }, "Place365": { - "hit@1": 0.359, - "hit@5": 0.636, - "hit@10": 0.728, - "ndcg_linear@1": 0.359, - "ndcg_linear@5": 0.5085200093794449, - "ndcg_linear@10": 0.5380972787675244, - "ndcg_exponential@1": 0.359, - "ndcg_exponential@5": 0.5085200093794449, - "ndcg_exponential@10": 0.5380972787675244, - "precision@1": 0.359, - "precision@5": 0.12719999999999998, - "precision@10": 0.0728, - "recall@1": 0.359, - "recall@5": 0.636, - "recall@10": 0.728, - "f1@1": 0.359, - "f1@5": 0.21200000000000005, - "f1@10": 0.13236363636363638, - "map@1": 0.359, - "map@5": 0.4659, - "map@10": 0.478006746031746, - "mrr@1": 0.359, - "mrr@5": 0.4659, - "mrr@10": 0.478006746031746, + "hit@1": 0.361, + "hit@5": 0.637, + "hit@10": 0.733, + "ndcg_linear@1": 0.361, + "ndcg_linear@5": 0.5092078481690382, + "ndcg_linear@10": 0.5399713957308175, + "ndcg_exponential@1": 0.361, + "ndcg_exponential@5": 0.5092078481690382, + "ndcg_exponential@10": 0.5399713957308175, + "precision@1": 0.361, + "precision@5": 0.12739999999999999, + "precision@10": 0.0733, + "recall@1": 0.361, + "recall@5": 0.637, + "recall@10": 0.733, + "f1@1": 0.361, + "f1@5": 0.2123333333333334, + "f1@10": 0.13327272727272726, + "map@1": 0.361, + "map@5": 0.46656666666666663, + "map@10": 0.47909682539682535, + "mrr@1": 0.361, + "mrr@5": 0.46656666666666663, + "mrr@10": 0.47909682539682535, "num_pred": 1000, "num_data": 1000 }, "ImageNet-A": { - "hit@1": 0.474, + "hit@1": 0.476, "hit@5": 0.706, - "hit@10": 0.787, - "ndcg_linear@1": 0.474, - "ndcg_linear@5": 0.5955931729705578, - "ndcg_linear@10": 0.6222176191859202, - "ndcg_exponential@1": 0.474, - "ndcg_exponential@5": 0.5955931729705578, - "ndcg_exponential@10": 0.6222176191859202, - "precision@1": 0.474, + "hit@10": 0.793, + "ndcg_linear@1": 0.476, + "ndcg_linear@5": 0.5955669488009249, + "ndcg_linear@10": 0.6238589315452278, + "ndcg_exponential@1": 0.476, + "ndcg_exponential@5": 0.5955669488009249, + "ndcg_exponential@10": 0.6238589315452278, + "precision@1": 0.476, "precision@5": 0.1412, - "precision@10": 0.07869999999999999, - "recall@1": 0.474, + "precision@10": 0.0793, + "recall@1": 0.476, "recall@5": 0.706, - "recall@10": 0.787, - "f1@1": 0.474, + "recall@10": 0.793, + "f1@1": 0.476, "f1@5": 0.23533333333333337, - "f1@10": 0.1430909090909091, - "map@1": 0.474, - "map@5": 0.559, - "map@10": 0.5702273809523809, - "mrr@1": 0.474, - "mrr@5": 0.559, - "mrr@10": 0.5702273809523809, + "f1@10": 0.1441818181818182, + "map@1": 0.476, + "map@5": 0.5590333333333333, + "map@10": 0.5707940476190476, + "mrr@1": 0.476, + "mrr@5": 0.5590333333333333, + "mrr@10": 0.5707940476190476, "num_pred": 1000, "num_data": 1000 }, "ImageNet-R": { "hit@1": 0.893, "hit@5": 0.968, - "hit@10": 0.98, + "hit@10": 0.981, "ndcg_linear@1": 0.893, - "ndcg_linear@5": 0.9354717686069736, - "ndcg_linear@10": 0.939360041159753, + "ndcg_linear@5": 0.9357158455533103, + "ndcg_linear@10": 0.9399110513889555, "ndcg_exponential@1": 0.893, - "ndcg_exponential@5": 0.9354717686069736, - "ndcg_exponential@10": 0.939360041159753, + "ndcg_exponential@5": 0.9357158455533103, + "ndcg_exponential@10": 0.9399110513889555, "precision@1": 0.893, "precision@5": 0.19360000000000002, - "precision@10": 0.098, + "precision@10": 0.0981, "recall@1": 0.893, "recall@5": 0.968, - "recall@10": 0.98, + "recall@10": 0.981, "f1@1": 0.893, "f1@5": 0.3226666666666666, - "f1@10": 0.17818181818181816, + "f1@10": 0.1783636363636363, "map@1": 0.893, - "map@5": 0.9244166666666668, - "map@10": 0.9260246031746031, + "map@5": 0.9247166666666667, + "map@10": 0.9264424603174604, "mrr@1": 0.893, - "mrr@5": 0.9244166666666668, - "mrr@10": 0.9260246031746031, + "mrr@5": 0.9247166666666667, + "mrr@10": 0.9264424603174604, "num_pred": 1000, "num_data": 1000 }, "ObjectNet": { - "hit@1": 0.652, + "hit@1": 0.651, "hit@5": 0.861, - "hit@10": 0.9, - "ndcg_linear@1": 0.652, - "ndcg_linear@5": 0.7682237991543275, - "ndcg_linear@10": 0.7806771789358414, - "ndcg_exponential@1": 0.652, - "ndcg_exponential@5": 0.7682237991543275, - "ndcg_exponential@10": 0.7806771789358414, - "precision@1": 0.652, - "precision@5": 0.1722, - "precision@10": 0.09000000000000001, - "recall@1": 0.652, + "hit@10": 0.903, + "ndcg_linear@1": 0.651, + "ndcg_linear@5": 0.7677325992824995, + "ndcg_linear@10": 0.7812331133508293, + "ndcg_exponential@1": 0.651, + "ndcg_exponential@5": 0.7677325992824995, + "ndcg_exponential@10": 0.7812331133508293, + "precision@1": 0.651, + "precision@5": 0.17220000000000002, + "precision@10": 0.0903, + "recall@1": 0.651, "recall@5": 0.861, - "recall@10": 0.9, - "f1@1": 0.652, - "f1@5": 0.287, - "f1@10": 0.16363636363636364, - "map@1": 0.652, - "map@5": 0.7369, - "map@10": 0.7419404761904761, - "mrr@1": 0.652, - "mrr@5": 0.7369, - "mrr@10": 0.7419404761904761, + "recall@10": 0.903, + "f1@1": 0.651, + "f1@5": 0.28700000000000003, + "f1@10": 0.16418181818181818, + "map@1": 0.651, + "map@5": 0.7362000000000001, + "map@10": 0.7417242063492062, + "mrr@1": 0.651, + "mrr@5": 0.7362000000000001, + "mrr@10": 0.7417242063492062, "num_pred": 1000, "num_data": 1000 }, "Country211": { - "hit@1": 0.252, + "hit@1": 0.258, "hit@5": 0.442, - "hit@10": 0.531, - "ndcg_linear@1": 0.252, - "ndcg_linear@5": 0.35154694867048564, - "ndcg_linear@10": 0.3803077107107121, - "ndcg_exponential@1": 0.252, - "ndcg_exponential@5": 0.35154694867048564, - "ndcg_exponential@10": 0.3803077107107121, - "precision@1": 0.252, - "precision@5": 0.0884, - "precision@10": 0.053099999999999994, - "recall@1": 0.252, + "hit@10": 0.536, + "ndcg_linear@1": 0.258, + "ndcg_linear@5": 0.35363815752576716, + "ndcg_linear@10": 0.3839202342317084, + "ndcg_exponential@1": 0.258, + "ndcg_exponential@5": 0.35363815752576716, + "ndcg_exponential@10": 0.3839202342317084, + "precision@1": 0.258, + "precision@5": 0.08839999999999999, + "precision@10": 0.053599999999999995, + "recall@1": 0.258, "recall@5": 0.442, - "recall@10": 0.531, - "f1@1": 0.252, + "recall@10": 0.536, + "f1@1": 0.258, "f1@5": 0.14733333333333337, - "f1@10": 0.09654545454545456, - "map@1": 0.252, - "map@5": 0.32154999999999995, - "map@10": 0.33340992063492064, - "mrr@1": 0.252, - "mrr@5": 0.32154999999999995, - "mrr@10": 0.33340992063492064, + "f1@10": 0.09745454545454547, + "map@1": 0.258, + "map@5": 0.3243833333333333, + "map@10": 0.33681388888888886, + "mrr@1": 0.258, + "mrr@5": 0.3243833333333333, + "mrr@10": 0.33681388888888886, "num_pred": 1000, "num_data": 1000 }, "OK-VQA": { - "hit@1": 0.515, - "hit@5": 0.777, - "hit@10": 0.844, - "ndcg_linear@1": 0.515, - "ndcg_linear@5": 0.6561916901050675, - "ndcg_linear@10": 0.6779083028403337, - "ndcg_exponential@1": 0.515, - "ndcg_exponential@5": 0.6561916901050675, - "ndcg_exponential@10": 0.6779083028403337, - "precision@1": 0.515, - "precision@5": 0.15539999999999998, - "precision@10": 0.0844, - "recall@1": 0.515, - "recall@5": 0.777, - "recall@10": 0.844, - "f1@1": 0.515, - "f1@5": 0.259, - "f1@10": 0.15345454545454545, - "map@1": 0.515, - "map@5": 0.6158166666666667, - "map@10": 0.6248087301587302, - "mrr@1": 0.515, - "mrr@5": 0.6158166666666667, - "mrr@10": 0.6248087301587302, + "hit@1": 0.517, + "hit@5": 0.771, + "hit@10": 0.841, + "ndcg_linear@1": 0.517, + "ndcg_linear@5": 0.6542491413151724, + "ndcg_linear@10": 0.6771926173196267, + "ndcg_exponential@1": 0.517, + "ndcg_exponential@5": 0.6542491413151724, + "ndcg_exponential@10": 0.6771926173196267, + "precision@1": 0.517, + "precision@5": 0.15419999999999998, + "precision@10": 0.08410000000000001, + "recall@1": 0.517, + "recall@5": 0.771, + "recall@10": 0.841, + "f1@1": 0.517, + "f1@5": 0.25700000000000006, + "f1@10": 0.1529090909090909, + "map@1": 0.517, + "map@5": 0.61515, + "map@10": 0.6247944444444444, + "mrr@1": 0.517, + "mrr@5": 0.61515, + "mrr@10": 0.6247944444444444, "num_pred": 1000, "num_data": 1000 }, "A-OKVQA": { - "hit@1": 0.436, - "hit@5": 0.727, - "hit@10": 0.799, - "ndcg_linear@1": 0.436, - "ndcg_linear@5": 0.5940223844793209, - "ndcg_linear@10": 0.6174487179188798, - "ndcg_exponential@1": 0.436, - "ndcg_exponential@5": 0.5940223844793209, - "ndcg_exponential@10": 0.6174487179188798, - "precision@1": 0.436, - "precision@5": 0.14539999999999997, - "precision@10": 0.0799, - "recall@1": 0.436, - "recall@5": 0.727, - "recall@10": 0.799, - "f1@1": 0.436, - "f1@5": 0.24233333333333337, - "f1@10": 0.14527272727272728, - "map@1": 0.436, - "map@5": 0.5494666666666667, - "map@10": 0.559218253968254, - "mrr@1": 0.436, - "mrr@5": 0.5494666666666667, - "mrr@10": 0.559218253968254, + "hit@1": 0.44, + "hit@5": 0.725, + "hit@10": 0.8, + "ndcg_linear@1": 0.44, + "ndcg_linear@5": 0.5943239119606247, + "ndcg_linear@10": 0.618788874740671, + "ndcg_exponential@1": 0.44, + "ndcg_exponential@5": 0.5943239119606247, + "ndcg_exponential@10": 0.618788874740671, + "precision@1": 0.44, + "precision@5": 0.145, + "precision@10": 0.08, + "recall@1": 0.44, + "recall@5": 0.725, + "recall@10": 0.8, + "f1@1": 0.44, + "f1@5": 0.24166666666666672, + "f1@10": 0.14545454545454548, + "map@1": 0.44, + "map@5": 0.5505833333333332, + "map@10": 0.5608083333333334, + "mrr@1": 0.44, + "mrr@5": 0.5505833333333332, + "mrr@10": 0.5608083333333334, "num_pred": 1000, "num_data": 1000 }, "DocVQA": { "hit@1": 0.901, - "hit@5": 0.97, + "hit@5": 0.971, "hit@10": 0.98, "ndcg_linear@1": 0.901, - "ndcg_linear@5": 0.9408983161880025, - "ndcg_linear@10": 0.9442465730917498, + "ndcg_linear@5": 0.9417223235057354, + "ndcg_linear@10": 0.9447143732223746, "ndcg_exponential@1": 0.901, - "ndcg_exponential@5": 0.9408983161880025, - "ndcg_exponential@10": 0.9442465730917498, + "ndcg_exponential@5": 0.9417223235057354, + "ndcg_exponential@10": 0.9447143732223746, "precision@1": 0.901, - "precision@5": 0.19400000000000003, + "precision@5": 0.1942, "precision@10": 0.09800000000000002, "recall@1": 0.901, - "recall@5": 0.97, + "recall@5": 0.971, "recall@10": 0.98, "f1@1": 0.901, - "f1@5": 0.32333333333333336, + "f1@5": 0.3236666666666666, "f1@10": 0.17818181818181816, "map@1": 0.901, - "map@5": 0.9309333333333333, - "map@10": 0.9323825396825398, + "map@5": 0.9316666666666668, + "map@10": 0.9329492063492063, "mrr@1": 0.901, - "mrr@5": 0.9309333333333333, - "mrr@10": 0.9323825396825398, + "mrr@5": 0.9316666666666668, + "mrr@10": 0.9329492063492063, "num_pred": 1000, "num_data": 1000 }, "InfographicsVQA": { - "hit@1": 0.588, - "hit@5": 0.811, - "hit@10": 0.869, - "ndcg_linear@1": 0.588, - "ndcg_linear@5": 0.7072011349880222, - "ndcg_linear@10": 0.7260241531938669, - "ndcg_exponential@1": 0.588, - "ndcg_exponential@5": 0.7072011349880222, - "ndcg_exponential@10": 0.7260241531938669, - "precision@1": 0.588, - "precision@5": 0.16219999999999998, - "precision@10": 0.0869, - "recall@1": 0.588, - "recall@5": 0.811, - "recall@10": 0.869, - "f1@1": 0.588, - "f1@5": 0.27033333333333337, - "f1@10": 0.158, - "map@1": 0.588, - "map@5": 0.6726333333333334, - "map@10": 0.680443253968254, - "mrr@1": 0.588, - "mrr@5": 0.6726333333333334, - "mrr@10": 0.680443253968254, + "hit@1": 0.591, + "hit@5": 0.812, + "hit@10": 0.866, + "ndcg_linear@1": 0.591, + "ndcg_linear@5": 0.7089653166709122, + "ndcg_linear@10": 0.7264764888179316, + "ndcg_exponential@1": 0.591, + "ndcg_exponential@5": 0.7089653166709122, + "ndcg_exponential@10": 0.7264764888179316, + "precision@1": 0.591, + "precision@5": 0.16240000000000002, + "precision@10": 0.0866, + "recall@1": 0.591, + "recall@5": 0.812, + "recall@10": 0.866, + "f1@1": 0.591, + "f1@5": 0.2706666666666667, + "f1@10": 0.15745454545454546, + "map@1": 0.591, + "map@5": 0.67465, + "map@10": 0.6819047619047619, + "mrr@1": 0.591, + "mrr@5": 0.67465, + "mrr@10": 0.6819047619047619, "num_pred": 1000, "num_data": 1000 }, "ChartQA": { - "hit@1": 0.474, - "hit@5": 0.676, - "hit@10": 0.753, - "ndcg_linear@1": 0.474, - "ndcg_linear@5": 0.5825057091236183, - "ndcg_linear@10": 0.607378542448543, - "ndcg_exponential@1": 0.474, - "ndcg_exponential@5": 0.5825057091236183, - "ndcg_exponential@10": 0.607378542448543, - "precision@1": 0.474, - "precision@5": 0.1352, - "precision@10": 0.0753, - "recall@1": 0.474, - "recall@5": 0.676, - "recall@10": 0.753, - "f1@1": 0.474, - "f1@5": 0.22533333333333336, - "f1@10": 0.1369090909090909, - "map@1": 0.474, - "map@5": 0.5513666666666666, - "map@10": 0.5616107142857143, - "mrr@1": 0.474, - "mrr@5": 0.5513666666666666, - "mrr@10": 0.5616107142857143, + "hit@1": 0.481, + "hit@5": 0.677, + "hit@10": 0.746, + "ndcg_linear@1": 0.481, + "ndcg_linear@5": 0.5855382014664426, + "ndcg_linear@10": 0.6079666085497525, + "ndcg_exponential@1": 0.481, + "ndcg_exponential@5": 0.5855382014664426, + "ndcg_exponential@10": 0.6079666085497525, + "precision@1": 0.481, + "precision@5": 0.1354, + "precision@10": 0.07460000000000001, + "recall@1": 0.481, + "recall@5": 0.677, + "recall@10": 0.746, + "f1@1": 0.481, + "f1@5": 0.22566666666666668, + "f1@10": 0.13563636363636367, + "map@1": 0.481, + "map@5": 0.5551333333333335, + "map@10": 0.5644496031746032, + "mrr@1": 0.481, + "mrr@5": 0.5551333333333335, + "mrr@10": 0.5644496031746032, "num_pred": 1000, "num_data": 1000 }, "Visual7W": { - "hit@1": 0.529, - "hit@5": 0.877, + "hit@1": 0.528, + "hit@5": 0.876, "hit@10": 0.953, - "ndcg_linear@1": 0.529, - "ndcg_linear@5": 0.7205350870379457, - "ndcg_linear@10": 0.7455710489687404, - "ndcg_exponential@1": 0.529, - "ndcg_exponential@5": 0.7205350870379457, - "ndcg_exponential@10": 0.7455710489687404, - "precision@1": 0.529, - "precision@5": 0.1754, + "ndcg_linear@1": 0.528, + "ndcg_linear@5": 0.7197281646021072, + "ndcg_linear@10": 0.745255014509615, + "ndcg_exponential@1": 0.528, + "ndcg_exponential@5": 0.7197281646021072, + "ndcg_exponential@10": 0.745255014509615, + "precision@1": 0.528, + "precision@5": 0.17520000000000002, "precision@10": 0.09530000000000001, - "recall@1": 0.529, - "recall@5": 0.877, + "recall@1": 0.528, + "recall@5": 0.876, "recall@10": 0.953, - "f1@1": 0.529, - "f1@5": 0.2923333333333334, + "f1@1": 0.528, + "f1@5": 0.292, "f1@10": 0.17327272727272724, - "map@1": 0.529, - "map@5": 0.6679166666666665, - "map@10": 0.6785115079365078, - "mrr@1": 0.529, - "mrr@5": 0.6679166666666665, - "mrr@10": 0.6785115079365078, + "map@1": 0.528, + "map@5": 0.66715, + "map@10": 0.6780543650793651, + "mrr@1": 0.528, + "mrr@5": 0.66715, + "mrr@10": 0.6780543650793651, "num_pred": 1000, "num_data": 1000 }, "ScienceQA": { - "hit@1": 0.382, - "hit@5": 0.662, - "hit@10": 0.771, - "ndcg_linear@1": 0.382, - "ndcg_linear@5": 0.5275673656588401, - "ndcg_linear@10": 0.5626365394128282, - "ndcg_exponential@1": 0.382, - "ndcg_exponential@5": 0.5275673656588401, - "ndcg_exponential@10": 0.5626365394128282, - "precision@1": 0.382, - "precision@5": 0.1324, - "precision@10": 0.07709999999999999, - "recall@1": 0.382, - "recall@5": 0.662, - "recall@10": 0.771, - "f1@1": 0.382, - "f1@5": 0.2206666666666667, - "f1@10": 0.14018181818181819, - "map@1": 0.382, - "map@5": 0.4832666666666667, - "map@10": 0.49763412698412696, - "mrr@1": 0.382, - "mrr@5": 0.4832666666666667, - "mrr@10": 0.49763412698412696, + "hit@1": 0.381, + "hit@5": 0.658, + "hit@10": 0.77, + "ndcg_linear@1": 0.381, + "ndcg_linear@5": 0.5260014741901842, + "ndcg_linear@10": 0.5622024629539769, + "ndcg_exponential@1": 0.381, + "ndcg_exponential@5": 0.5260014741901842, + "ndcg_exponential@10": 0.5622024629539769, + "precision@1": 0.381, + "precision@5": 0.1316, + "precision@10": 0.077, + "recall@1": 0.381, + "recall@5": 0.658, + "recall@10": 0.77, + "f1@1": 0.381, + "f1@5": 0.2193333333333334, + "f1@10": 0.14, + "map@1": 0.381, + "map@5": 0.48236666666666667, + "map@10": 0.49729325396825397, + "mrr@1": 0.381, + "mrr@5": 0.48236666666666667, + "mrr@10": 0.49729325396825397, "num_pred": 1000, "num_data": 1000 }, "VizWiz": { "hit@1": 0.433, - "hit@5": 0.61, - "hit@10": 0.671, + "hit@5": 0.612, + "hit@10": 0.672, "ndcg_linear@1": 0.433, - "ndcg_linear@5": 0.5297118471987433, - "ndcg_linear@10": 0.5497377057138165, + "ndcg_linear@5": 0.5305471591248572, + "ndcg_linear@10": 0.5501507512245414, "ndcg_exponential@1": 0.433, - "ndcg_exponential@5": 0.5297118471987433, - "ndcg_exponential@10": 0.5497377057138165, + "ndcg_exponential@5": 0.5305471591248572, + "ndcg_exponential@10": 0.5501507512245414, "precision@1": 0.433, - "precision@5": 0.12199999999999997, - "precision@10": 0.06709999999999999, + "precision@5": 0.1224, + "precision@10": 0.06719999999999998, "recall@1": 0.433, - "recall@5": 0.61, - "recall@10": 0.671, + "recall@5": 0.612, + "recall@10": 0.672, "f1@1": 0.433, - "f1@5": 0.20333333333333337, - "f1@10": 0.12200000000000003, + "f1@5": 0.20400000000000007, + "f1@10": 0.1221818181818182, "map@1": 0.433, - "map@5": 0.5027333333333334, - "map@10": 0.5111761904761905, + "map@5": 0.5032166666666666, + "map@10": 0.5114214285714286, "mrr@1": 0.433, - "mrr@5": 0.5027333333333334, - "mrr@10": 0.5111761904761905, + "mrr@5": 0.5032166666666666, + "mrr@10": 0.5114214285714286, "num_pred": 1000, "num_data": 1000 }, "GQA": { - "hit@1": 0.649, - "hit@5": 0.886, + "hit@1": 0.654, + "hit@5": 0.885, "hit@10": 0.927, - "ndcg_linear@1": 0.649, - "ndcg_linear@5": 0.7797599318993801, - "ndcg_linear@10": 0.7931564189677488, - "ndcg_exponential@1": 0.649, - "ndcg_exponential@5": 0.7797599318993801, - "ndcg_exponential@10": 0.7931564189677488, - "precision@1": 0.649, - "precision@5": 0.17720000000000005, + "ndcg_linear@1": 0.654, + "ndcg_linear@5": 0.781604043953666, + "ndcg_linear@10": 0.7952750023722877, + "ndcg_exponential@1": 0.654, + "ndcg_exponential@5": 0.781604043953666, + "ndcg_exponential@10": 0.7952750023722877, + "precision@1": 0.654, + "precision@5": 0.17700000000000002, "precision@10": 0.09270000000000002, - "recall@1": 0.649, - "recall@5": 0.886, + "recall@1": 0.654, + "recall@5": 0.885, "recall@10": 0.927, - "f1@1": 0.649, - "f1@5": 0.2953333333333334, - "f1@10": 0.16854545454545455, - "map@1": 0.649, - "map@5": 0.7439666666666667, - "map@10": 0.7495777777777778, - "mrr@1": 0.649, - "mrr@5": 0.7439666666666667, - "mrr@10": 0.7495777777777778, + "f1@1": 0.654, + "f1@5": 0.29500000000000004, + "f1@10": 0.16854545454545453, + "map@1": 0.654, + "map@5": 0.74675, + "map@10": 0.7524496031746032, + "mrr@1": 0.654, + "mrr@5": 0.74675, + "mrr@10": 0.7524496031746032, "num_pred": 1000, "num_data": 1000 }, "TextVQA": { - "hit@1": 0.722, - "hit@5": 0.833, - "hit@10": 0.863, - "ndcg_linear@1": 0.722, - "ndcg_linear@5": 0.7819030121133494, - "ndcg_linear@10": 0.7918381023735301, - "ndcg_exponential@1": 0.722, - "ndcg_exponential@5": 0.7819030121133494, - "ndcg_exponential@10": 0.7918381023735301, - "precision@1": 0.722, - "precision@5": 0.16660000000000003, - "precision@10": 0.08630000000000002, - "recall@1": 0.722, - "recall@5": 0.833, - "recall@10": 0.863, - "f1@1": 0.722, - "f1@5": 0.2776666666666667, - "f1@10": 0.15690909090909091, - "map@1": 0.722, - "map@5": 0.7648166666666666, - "map@10": 0.7690543650793651, - "mrr@1": 0.722, - "mrr@5": 0.7648166666666666, - "mrr@10": 0.7690543650793651, + "hit@1": 0.716, + "hit@5": 0.832, + "hit@10": 0.862, + "ndcg_linear@1": 0.716, + "ndcg_linear@5": 0.7791630909436905, + "ndcg_linear@10": 0.7890583367835231, + "ndcg_exponential@1": 0.716, + "ndcg_exponential@5": 0.7791630909436905, + "ndcg_exponential@10": 0.7890583367835231, + "precision@1": 0.716, + "precision@5": 0.1664, + "precision@10": 0.0862, + "recall@1": 0.716, + "recall@5": 0.832, + "recall@10": 0.862, + "f1@1": 0.716, + "f1@5": 0.2773333333333334, + "f1@10": 0.15672727272727274, + "map@1": 0.716, + "map@5": 0.7614500000000001, + "map@10": 0.7656468253968253, + "mrr@1": 0.716, + "mrr@5": 0.7614500000000001, + "mrr@10": 0.7656468253968253, "num_pred": 1000, "num_data": 1000 }, "VisDial": { "hit@1": 0.827, - "hit@5": 0.968, + "hit@5": 0.97, "hit@10": 0.986, "ndcg_linear@1": 0.827, - "ndcg_linear@5": 0.9045672377369148, - "ndcg_linear@10": 0.9104807612339774, + "ndcg_linear@5": 0.9055085212244576, + "ndcg_linear@10": 0.9107435714804505, "ndcg_exponential@1": 0.827, - "ndcg_exponential@5": 0.9045672377369148, - "ndcg_exponential@10": 0.9104807612339774, + "ndcg_exponential@5": 0.9055085212244576, + "ndcg_exponential@10": 0.9107435714804505, "precision@1": 0.827, - "precision@5": 0.19360000000000002, + "precision@5": 0.19400000000000003, "precision@10": 0.09860000000000002, "recall@1": 0.827, - "recall@5": 0.968, + "recall@5": 0.97, "recall@10": 0.986, "f1@1": 0.827, - "f1@5": 0.3226666666666667, + "f1@5": 0.32333333333333336, "f1@10": 0.17927272727272725, "map@1": 0.827, - "map@5": 0.88325, - "map@10": 0.8857444444444444, + "map@5": 0.8838499999999999, + "map@10": 0.8860452380952382, "mrr@1": 0.827, - "mrr@5": 0.88325, - "mrr@10": 0.8857444444444444, + "mrr@5": 0.8838499999999999, + "mrr@10": 0.8860452380952382, "num_pred": 1000, "num_data": 1000 }, "CIRR": { - "hit@1": 0.575, - "hit@5": 0.899, - "hit@10": 0.953, - "ndcg_linear@1": 0.575, - "ndcg_linear@5": 0.7551683396424022, - "ndcg_linear@10": 0.7727898661723096, - "ndcg_exponential@1": 0.575, - "ndcg_exponential@5": 0.7551683396424022, - "ndcg_exponential@10": 0.7727898661723096, - "precision@1": 0.575, - "precision@5": 0.17980000000000002, - "precision@10": 0.09530000000000001, - "recall@1": 0.575, - "recall@5": 0.899, - "recall@10": 0.953, - "f1@1": 0.575, - "f1@5": 0.2996666666666667, - "f1@10": 0.17327272727272727, - "map@1": 0.575, - "map@5": 0.7066166666666667, - "map@10": 0.7139825396825397, - "mrr@1": 0.575, - "mrr@5": 0.7066166666666667, - "mrr@10": 0.7139825396825397, + "hit@1": 0.573, + "hit@5": 0.898, + "hit@10": 0.955, + "ndcg_linear@1": 0.573, + "ndcg_linear@5": 0.7543312470394863, + "ndcg_linear@10": 0.7729363843133801, + "ndcg_exponential@1": 0.573, + "ndcg_exponential@5": 0.7543312470394863, + "ndcg_exponential@10": 0.7729363843133801, + "precision@1": 0.573, + "precision@5": 0.1796, + "precision@10": 0.09550000000000002, + "recall@1": 0.573, + "recall@5": 0.898, + "recall@10": 0.955, + "f1@1": 0.573, + "f1@5": 0.2993333333333334, + "f1@10": 0.17363636363636362, + "map@1": 0.573, + "map@5": 0.7057666666666667, + "map@10": 0.7135480158730159, + "mrr@1": 0.573, + "mrr@5": 0.7057666666666667, + "mrr@10": 0.7135480158730159, "num_pred": 1000, "num_data": 1000 }, "VisualNews_t2i": { - "hit@1": 0.745, - "hit@5": 0.895, - "hit@10": 0.928, - "ndcg_linear@1": 0.745, - "ndcg_linear@5": 0.8286638944708601, - "ndcg_linear@10": 0.8393509120511464, - "ndcg_exponential@1": 0.745, - "ndcg_exponential@5": 0.8286638944708601, - "ndcg_exponential@10": 0.8393509120511464, - "precision@1": 0.745, - "precision@5": 0.17900000000000002, - "precision@10": 0.09280000000000001, - "recall@1": 0.745, - "recall@5": 0.895, - "recall@10": 0.928, - "f1@1": 0.745, - "f1@5": 0.2983333333333334, - "f1@10": 0.1687272727272727, - "map@1": 0.745, - "map@5": 0.8062833333333334, - "map@10": 0.8107011904761905, - "mrr@1": 0.745, - "mrr@5": 0.8062833333333334, - "mrr@10": 0.8107011904761905, + "hit@1": 0.747, + "hit@5": 0.893, + "hit@10": 0.927, + "ndcg_linear@1": 0.747, + "ndcg_linear@5": 0.8284713584056438, + "ndcg_linear@10": 0.8396189416474574, + "ndcg_exponential@1": 0.747, + "ndcg_exponential@5": 0.8284713584056438, + "ndcg_exponential@10": 0.8396189416474574, + "precision@1": 0.747, + "precision@5": 0.17860000000000004, + "precision@10": 0.09270000000000002, + "recall@1": 0.747, + "recall@5": 0.893, + "recall@10": 0.927, + "f1@1": 0.747, + "f1@5": 0.2976666666666667, + "f1@10": 0.16854545454545453, + "map@1": 0.747, + "map@5": 0.8067000000000001, + "map@10": 0.8113916666666666, + "mrr@1": 0.747, + "mrr@5": 0.8067000000000001, + "mrr@10": 0.8113916666666666, "num_pred": 1000, "num_data": 1000 }, "VisualNews_i2t": { - "hit@1": 0.782, + "hit@1": 0.783, "hit@5": 0.906, - "hit@10": 0.935, - "ndcg_linear@1": 0.782, - "ndcg_linear@5": 0.8486488958609838, - "ndcg_linear@10": 0.8583163159480872, - "ndcg_exponential@1": 0.782, - "ndcg_exponential@5": 0.8486488958609838, - "ndcg_exponential@10": 0.8583163159480872, - "precision@1": 0.782, + "hit@10": 0.938, + "ndcg_linear@1": 0.783, + "ndcg_linear@5": 0.8489924664163245, + "ndcg_linear@10": 0.8594319667493224, + "ndcg_exponential@1": 0.783, + "ndcg_exponential@5": 0.8489924664163245, + "ndcg_exponential@10": 0.8594319667493224, + "precision@1": 0.783, "precision@5": 0.18120000000000003, - "precision@10": 0.09350000000000001, - "recall@1": 0.782, + "precision@10": 0.09380000000000001, + "recall@1": 0.783, "recall@5": 0.906, - "recall@10": 0.935, - "f1@1": 0.782, + "recall@10": 0.938, + "f1@1": 0.783, "f1@5": 0.302, - "f1@10": 0.17, - "map@1": 0.782, - "map@5": 0.8295, - "map@10": 0.833656746031746, - "mrr@1": 0.782, - "mrr@5": 0.8295, - "mrr@10": 0.833656746031746, + "f1@10": 0.17054545454545453, + "map@1": 0.783, + "map@5": 0.8299666666666667, + "map@10": 0.8343297619047619, + "mrr@1": 0.783, + "mrr@5": 0.8299666666666667, + "mrr@10": 0.8343297619047619, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_t2i": { - "hit@1": 0.753, - "hit@5": 0.949, + "hit@1": 0.759, + "hit@5": 0.948, "hit@10": 0.981, - "ndcg_linear@1": 0.753, - "ndcg_linear@5": 0.8611598177428067, - "ndcg_linear@10": 0.8714056173186473, - "ndcg_exponential@1": 0.753, - "ndcg_exponential@5": 0.8611598177428067, - "ndcg_exponential@10": 0.8714056173186473, - "precision@1": 0.753, - "precision@5": 0.18980000000000002, + "ndcg_linear@1": 0.759, + "ndcg_linear@5": 0.8630928164766271, + "ndcg_linear@10": 0.8736575145046793, + "ndcg_exponential@1": 0.759, + "ndcg_exponential@5": 0.8630928164766271, + "ndcg_exponential@10": 0.8736575145046793, + "precision@1": 0.759, + "precision@5": 0.18960000000000002, "precision@10": 0.0981, - "recall@1": 0.753, - "recall@5": 0.949, + "recall@1": 0.759, + "recall@5": 0.948, "recall@10": 0.981, - "f1@1": 0.753, - "f1@5": 0.3163333333333333, + "f1@1": 0.759, + "f1@5": 0.316, "f1@10": 0.1783636363636363, - "map@1": 0.753, - "map@5": 0.8315833333333333, - "map@10": 0.83575, - "mrr@1": 0.753, - "mrr@5": 0.8315833333333333, - "mrr@10": 0.83575, + "map@1": 0.759, + "map@5": 0.8345166666666667, + "map@10": 0.8388123015873016, + "mrr@1": 0.759, + "mrr@5": 0.8345166666666667, + "mrr@10": 0.8388123015873016, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_i2t": { - "hit@1": 0.714, - "hit@5": 0.94, - "hit@10": 0.987, - "ndcg_linear@1": 0.714, - "ndcg_linear@5": 0.8382480589924204, - "ndcg_linear@10": 0.8536803755951516, - "ndcg_exponential@1": 0.714, - "ndcg_exponential@5": 0.8382480589924204, - "ndcg_exponential@10": 0.8536803755951516, - "precision@1": 0.714, - "precision@5": 0.18800000000000003, - "precision@10": 0.09870000000000002, - "recall@1": 0.714, - "recall@5": 0.94, - "recall@10": 0.987, - "f1@1": 0.714, - "f1@5": 0.31333333333333335, - "f1@10": 0.17945454545454545, - "map@1": 0.714, - "map@5": 0.8040333333333333, - "map@10": 0.8105345238095237, - "mrr@1": 0.714, - "mrr@5": 0.8040333333333333, - "mrr@10": 0.8105345238095237, + "hit@1": 0.711, + "hit@5": 0.938, + "hit@10": 0.985, + "ndcg_linear@1": 0.711, + "ndcg_linear@5": 0.8360460251201203, + "ndcg_linear@10": 0.8516165753567416, + "ndcg_exponential@1": 0.711, + "ndcg_exponential@5": 0.8360460251201203, + "ndcg_exponential@10": 0.8516165753567416, + "precision@1": 0.711, + "precision@5": 0.18760000000000002, + "precision@10": 0.09850000000000002, + "recall@1": 0.711, + "recall@5": 0.938, + "recall@10": 0.985, + "f1@1": 0.711, + "f1@5": 0.31266666666666665, + "f1@10": 0.17909090909090908, + "map@1": 0.711, + "map@5": 0.80175, + "map@10": 0.8083920634920635, + "mrr@1": 0.711, + "mrr@5": 0.80175, + "mrr@10": 0.8083920634920635, "num_pred": 1000, "num_data": 1000 }, "NIGHTS": { - "hit@1": 0.686, - "hit@5": 0.98, + "hit@1": 0.684, + "hit@5": 0.981, "hit@10": 0.996, - "ndcg_linear@1": 0.686, - "ndcg_linear@5": 0.8539747226047097, - "ndcg_linear@10": 0.8594110413481301, - "ndcg_exponential@1": 0.686, - "ndcg_exponential@5": 0.8539747226047097, - "ndcg_exponential@10": 0.8594110413481301, - "precision@1": 0.686, - "precision@5": 0.196, + "ndcg_linear@1": 0.684, + "ndcg_linear@5": 0.8533023174005421, + "ndcg_linear@10": 0.8583595551030795, + "ndcg_exponential@1": 0.684, + "ndcg_exponential@5": 0.8533023174005421, + "ndcg_exponential@10": 0.8583595551030795, + "precision@1": 0.684, + "precision@5": 0.1962, "precision@10": 0.09960000000000002, - "recall@1": 0.686, - "recall@5": 0.98, + "recall@1": 0.684, + "recall@5": 0.981, "recall@10": 0.996, - "f1@1": 0.686, - "f1@5": 0.3266666666666666, + "f1@1": 0.684, + "f1@5": 0.327, "f1@10": 0.18109090909090905, - "map@1": 0.686, - "map@5": 0.81095, - "map@10": 0.813347619047619, - "mrr@1": 0.686, - "mrr@5": 0.81095, - "mrr@10": 0.813347619047619, + "map@1": 0.684, + "map@5": 0.8097666666666666, + "map@10": 0.8119738095238095, + "mrr@1": 0.684, + "mrr@5": 0.8097666666666666, + "mrr@10": 0.8119738095238095, "num_pred": 1000, "num_data": 1000 }, "WebQA": { "hit@1": 0.906, - "hit@5": 0.988, + "hit@5": 0.987, "hit@10": 0.993, "ndcg_linear@1": 0.906, - "ndcg_linear@5": 0.953453471346856, - "ndcg_linear@10": 0.9551658857210721, + "ndcg_linear@5": 0.9533900843584092, + "ndcg_linear@10": 0.9554637113169604, "ndcg_exponential@1": 0.906, - "ndcg_exponential@5": 0.953453471346856, - "ndcg_exponential@10": 0.9551658857210721, + "ndcg_exponential@5": 0.9533900843584092, + "ndcg_exponential@10": 0.9554637113169604, "precision@1": 0.906, - "precision@5": 0.19760000000000003, + "precision@5": 0.19740000000000005, "precision@10": 0.09930000000000001, "recall@1": 0.906, - "recall@5": 0.988, + "recall@5": 0.987, "recall@10": 0.993, "f1@1": 0.906, - "f1@5": 0.3293333333333333, + "f1@5": 0.329, "f1@10": 0.18054545454545454, "map@1": 0.906, - "map@5": 0.9415999999999999, - "map@10": 0.9423619047619047, + "map@5": 0.9418166666666666, + "map@10": 0.9427511904761905, "mrr@1": 0.906, - "mrr@5": 0.9415999999999999, - "mrr@10": 0.9423619047619047, + "mrr@5": 0.9418166666666666, + "mrr@10": 0.9427511904761905, "num_pred": 1000, "num_data": 1000 }, "FashionIQ": { - "hit@1": 0.195, + "hit@1": 0.196, "hit@5": 0.434, - "hit@10": 0.567, - "ndcg_linear@1": 0.195, - "ndcg_linear@5": 0.3199008192098037, - "ndcg_linear@10": 0.36332550239850325, - "ndcg_exponential@1": 0.195, - "ndcg_exponential@5": 0.3199008192098037, - "ndcg_exponential@10": 0.36332550239850325, - "precision@1": 0.195, - "precision@5": 0.08680000000000002, - "precision@10": 0.05669999999999999, - "recall@1": 0.195, + "hit@10": 0.566, + "ndcg_linear@1": 0.196, + "ndcg_linear@5": 0.3189575713907209, + "ndcg_linear@10": 0.36167203067306924, + "ndcg_exponential@1": 0.196, + "ndcg_exponential@5": 0.3189575713907209, + "ndcg_exponential@10": 0.36167203067306924, + "precision@1": 0.196, + "precision@5": 0.0868, + "precision@10": 0.0566, + "recall@1": 0.196, "recall@5": 0.434, - "recall@10": 0.567, - "f1@1": 0.195, + "recall@10": 0.566, + "f1@1": 0.196, "f1@5": 0.1446666666666667, - "f1@10": 0.10309090909090911, - "map@1": 0.195, - "map@5": 0.28218333333333334, - "map@10": 0.3003369047619047, - "mrr@1": 0.195, - "mrr@5": 0.28218333333333334, - "mrr@10": 0.3003369047619047, + "f1@10": 0.10290909090909094, + "map@1": 0.196, + "map@5": 0.28108333333333335, + "map@10": 0.2987297619047619, + "mrr@1": 0.196, + "mrr@5": 0.28108333333333335, + "mrr@10": 0.2987297619047619, "num_pred": 1000, "num_data": 1000 }, "Wiki-SS-NQ": { - "hit@1": 0.669, + "hit@1": 0.676, "hit@5": 0.88, - "hit@10": 0.921, - "ndcg_linear@1": 0.669, - "ndcg_linear@5": 0.787070983987401, - "ndcg_linear@10": 0.8003663875618333, - "ndcg_exponential@1": 0.669, - "ndcg_exponential@5": 0.787070983987401, - "ndcg_exponential@10": 0.8003663875618333, - "precision@1": 0.669, + "hit@10": 0.924, + "ndcg_linear@1": 0.676, + "ndcg_linear@5": 0.7895230044598843, + "ndcg_linear@10": 0.8038512926825455, + "ndcg_exponential@1": 0.676, + "ndcg_exponential@5": 0.7895230044598843, + "ndcg_exponential@10": 0.8038512926825455, + "precision@1": 0.676, "precision@5": 0.176, - "precision@10": 0.09210000000000002, - "recall@1": 0.669, + "precision@10": 0.09240000000000001, + "recall@1": 0.676, "recall@5": 0.88, - "recall@10": 0.921, - "f1@1": 0.669, + "recall@10": 0.924, + "f1@1": 0.676, "f1@5": 0.2933333333333334, - "f1@10": 0.16745454545454547, - "map@1": 0.669, - "map@5": 0.7556499999999999, - "map@10": 0.7611603174603174, - "mrr@1": 0.669, - "mrr@5": 0.7556499999999999, - "mrr@10": 0.7611603174603174, + "f1@10": 0.168, + "map@1": 0.676, + "map@5": 0.759, + "map@10": 0.7649746031746031, + "mrr@1": 0.676, + "mrr@5": 0.759, + "mrr@10": 0.7649746031746031, "num_pred": 1000, "num_data": 1000 }, "OVEN": { - "hit@1": 0.643, - "hit@5": 0.851, - "hit@10": 0.907, - "ndcg_linear@1": 0.643, - "ndcg_linear@5": 0.75896196577168, - "ndcg_linear@10": 0.7771824352969045, - "ndcg_exponential@1": 0.643, - "ndcg_exponential@5": 0.75896196577168, - "ndcg_exponential@10": 0.7771824352969045, - "precision@1": 0.643, - "precision@5": 0.1702, - "precision@10": 0.09070000000000002, - "recall@1": 0.643, - "recall@5": 0.851, - "recall@10": 0.907, - "f1@1": 0.643, - "f1@5": 0.28366666666666673, - "f1@10": 0.16490909090909087, - "map@1": 0.643, - "map@5": 0.7279166666666668, - "map@10": 0.7354976190476191, - "mrr@1": 0.643, - "mrr@5": 0.7279166666666668, - "mrr@10": 0.7354976190476191, + "hit@1": 0.648, + "hit@5": 0.849, + "hit@10": 0.909, + "ndcg_linear@1": 0.648, + "ndcg_linear@5": 0.760310363658115, + "ndcg_linear@10": 0.7799214968404775, + "ndcg_exponential@1": 0.648, + "ndcg_exponential@5": 0.760310363658115, + "ndcg_exponential@10": 0.7799214968404775, + "precision@1": 0.648, + "precision@5": 0.1698, + "precision@10": 0.09090000000000001, + "recall@1": 0.648, + "recall@5": 0.849, + "recall@10": 0.909, + "f1@1": 0.648, + "f1@5": 0.283, + "f1@10": 0.16527272727272727, + "map@1": 0.648, + "map@5": 0.7303666666666667, + "map@10": 0.7385805555555556, + "mrr@1": 0.648, + "mrr@5": 0.7303666666666667, + "mrr@10": 0.7385805555555556, "num_pred": 1000, "num_data": 1000 }, "EDIS": { - "hit@1": 0.841, - "hit@5": 0.982, + "hit@1": 0.842, + "hit@5": 0.98, "hit@10": 0.991, - "ndcg_linear@1": 0.841, - "ndcg_linear@5": 0.919465963451109, - "ndcg_linear@10": 0.9224571152777739, - "ndcg_exponential@1": 0.841, - "ndcg_exponential@5": 0.919465963451109, - "ndcg_exponential@10": 0.9224571152777739, - "precision@1": 0.841, - "precision@5": 0.19640000000000005, + "ndcg_linear@1": 0.842, + "ndcg_linear@5": 0.919148434085801, + "ndcg_linear@10": 0.9228893090215785, + "ndcg_exponential@1": 0.842, + "ndcg_exponential@5": 0.919148434085801, + "ndcg_exponential@10": 0.9228893090215785, + "precision@1": 0.842, + "precision@5": 0.19600000000000004, "precision@10": 0.09910000000000002, - "recall@1": 0.841, - "recall@5": 0.982, + "recall@1": 0.842, + "recall@5": 0.98, "recall@10": 0.991, - "f1@1": 0.841, - "f1@5": 0.3273333333333333, + "f1@1": 0.842, + "f1@5": 0.3266666666666666, "f1@10": 0.18018181818181817, - "map@1": 0.841, - "map@5": 0.8983833333333333, - "map@10": 0.8996650793650793, - "mrr@1": 0.841, - "mrr@5": 0.8983833333333333, - "mrr@10": 0.8996650793650793, + "map@1": 0.842, + "map@5": 0.8986, + "map@10": 0.9002527777777778, + "mrr@1": 0.842, + "mrr@5": 0.8986, + "mrr@10": 0.9002527777777778, "num_pred": 1000, "num_data": 1000 }, "MSCOCO": { - "hit@1": 0.671, - "hit@5": 0.796, - "hit@10": 0.846, - "ndcg_linear@1": 0.671, - "ndcg_linear@5": 0.7385700746119735, - "ndcg_linear@10": 0.75471044609582, - "ndcg_exponential@1": 0.671, - "ndcg_exponential@5": 0.7385700746119735, - "ndcg_exponential@10": 0.75471044609582, - "precision@1": 0.671, - "precision@5": 0.1592, - "precision@10": 0.0846, - "recall@1": 0.671, - "recall@5": 0.796, - "recall@10": 0.846, - "f1@1": 0.671, - "f1@5": 0.26533333333333337, - "f1@10": 0.15381818181818183, - "map@1": 0.671, - "map@5": 0.7193833333333334, - "map@10": 0.7260313492063492, - "mrr@1": 0.671, - "mrr@5": 0.7193833333333334, - "mrr@10": 0.7260313492063492, + "hit@1": 0.662, + "hit@5": 0.795, + "hit@10": 0.849, + "ndcg_linear@1": 0.662, + "ndcg_linear@5": 0.7346685120227207, + "ndcg_linear@10": 0.7519423615793314, + "ndcg_exponential@1": 0.662, + "ndcg_exponential@5": 0.7346685120227207, + "ndcg_exponential@10": 0.7519423615793314, + "precision@1": 0.662, + "precision@5": 0.159, + "precision@10": 0.0849, + "recall@1": 0.662, + "recall@5": 0.795, + "recall@10": 0.849, + "f1@1": 0.662, + "f1@5": 0.26500000000000007, + "f1@10": 0.15436363636363637, + "map@1": 0.662, + "map@5": 0.71445, + "map@10": 0.7214714285714287, + "mrr@1": 0.662, + "mrr@5": 0.71445, + "mrr@10": 0.7214714285714287, "num_pred": 1000, "num_data": 1000 }, "RefCOCO": { - "hit@1": 0.871, - "hit@5": 0.978, - "hit@10": 0.992, - "ndcg_linear@1": 0.871, - "ndcg_linear@5": 0.9320666707886123, - "ndcg_linear@10": 0.9367513120480281, - "ndcg_exponential@1": 0.871, - "ndcg_exponential@5": 0.9320666707886123, - "ndcg_exponential@10": 0.9367513120480281, - "precision@1": 0.871, - "precision@5": 0.19560000000000002, - "precision@10": 0.09920000000000001, - "recall@1": 0.871, - "recall@5": 0.978, - "recall@10": 0.992, - "f1@1": 0.871, - "f1@5": 0.326, - "f1@10": 0.18036363636363634, - "map@1": 0.871, - "map@5": 0.9164333333333334, - "map@10": 0.9184607142857143, - "mrr@1": 0.871, - "mrr@5": 0.9164333333333334, - "mrr@10": 0.9184607142857143, + "hit@1": 0.87, + "hit@5": 0.977, + "hit@10": 0.991, + "ndcg_linear@1": 0.87, + "ndcg_linear@5": 0.9313800711768758, + "ndcg_linear@10": 0.936074207893862, + "ndcg_exponential@1": 0.87, + "ndcg_exponential@5": 0.9313800711768758, + "ndcg_exponential@10": 0.936074207893862, + "precision@1": 0.87, + "precision@5": 0.19540000000000005, + "precision@10": 0.09910000000000002, + "recall@1": 0.87, + "recall@5": 0.977, + "recall@10": 0.991, + "f1@1": 0.87, + "f1@5": 0.32566666666666666, + "f1@10": 0.1801818181818182, + "map@1": 0.87, + "map@5": 0.9158166666666666, + "map@10": 0.9178523809523811, + "mrr@1": 0.87, + "mrr@5": 0.9158166666666666, + "mrr@10": 0.9178523809523811, "num_pred": 1000, "num_data": 1000 }, "RefCOCO-Matching": { - "hit@1": 0.858, - "hit@5": 1.0, + "hit@1": 0.863, + "hit@5": 0.999, "hit@10": 1.0, - "ndcg_linear@1": 0.858, - "ndcg_linear@5": 0.948647694865312, - "ndcg_linear@10": 0.948647694865312, - "ndcg_exponential@1": 0.858, - "ndcg_exponential@5": 0.948647694865312, - "ndcg_exponential@10": 0.948647694865312, - "precision@1": 0.858, - "precision@5": 0.20000000000000004, + "ndcg_linear@1": 0.863, + "ndcg_linear@5": 0.9501061932902203, + "ndcg_linear@10": 0.9504624004773283, + "ndcg_exponential@1": 0.863, + "ndcg_exponential@5": 0.9501061932902203, + "ndcg_exponential@10": 0.9504624004773283, + "precision@1": 0.863, + "precision@5": 0.19980000000000003, "precision@10": 0.10000000000000002, - "recall@1": 0.858, - "recall@5": 1.0, + "recall@1": 0.863, + "recall@5": 0.999, "recall@10": 1.0, - "f1@1": 0.858, - "f1@5": 0.3333333333333333, + "f1@1": 0.863, + "f1@5": 0.333, "f1@10": 0.18181818181818182, - "map@1": 0.858, - "map@5": 0.93095, - "map@10": 0.93095, - "mrr@1": 0.858, - "mrr@5": 0.92795, - "mrr@10": 0.92795, + "map@1": 0.863, + "map@5": 0.93325, + "map@10": 0.9334166666666668, + "mrr@1": 0.863, + "mrr@5": 0.93025, + "mrr@10": 0.9304166666666668, "num_pred": 1000, "num_data": 1000 }, "Visual7W-Pointing": { - "hit@1": 0.692, - "hit@5": 0.865, - "hit@10": 0.906, - "ndcg_linear@1": 0.692, - "ndcg_linear@5": 0.7886508353101727, - "ndcg_linear@10": 0.8018399920089694, - "ndcg_exponential@1": 0.692, - "ndcg_exponential@5": 0.7886508353101727, - "ndcg_exponential@10": 0.8018399920089694, - "precision@1": 0.692, - "precision@5": 0.173, - "precision@10": 0.09060000000000001, - "recall@1": 0.692, - "recall@5": 0.865, - "recall@10": 0.906, - "f1@1": 0.692, - "f1@5": 0.2883333333333334, - "f1@10": 0.16472727272727272, - "map@1": 0.692, - "map@5": 0.7628499999999999, - "map@10": 0.7682543650793651, - "mrr@1": 0.692, - "mrr@5": 0.7628499999999999, - "mrr@10": 0.7682543650793651, + "hit@1": 0.69, + "hit@5": 0.867, + "hit@10": 0.904, + "ndcg_linear@1": 0.69, + "ndcg_linear@5": 0.7885709049387766, + "ndcg_linear@10": 0.8005697685778882, + "ndcg_exponential@1": 0.69, + "ndcg_exponential@5": 0.7885709049387766, + "ndcg_exponential@10": 0.8005697685778882, + "precision@1": 0.69, + "precision@5": 0.17340000000000003, + "precision@10": 0.09040000000000001, + "recall@1": 0.69, + "recall@5": 0.867, + "recall@10": 0.904, + "f1@1": 0.69, + "f1@5": 0.289, + "f1@10": 0.16436363636363635, + "map@1": 0.69, + "map@5": 0.7620833333333332, + "map@10": 0.7670563492063492, + "mrr@1": 0.69, + "mrr@5": 0.7620833333333332, + "mrr@10": 0.7670563492063492, "num_pred": 1000, "num_data": 1000 } }, "video": { "K700": { - "hit@1": 0.38, + "hit@1": 0.382, "hit@5": 0.623, - "hit@10": 0.697, - "ndcg_linear@1": 0.38, - "ndcg_linear@5": 0.5087323881529896, - "ndcg_linear@10": 0.5328292310353233, - "ndcg_exponential@1": 0.38, - "ndcg_exponential@5": 0.5087323881529896, - "ndcg_exponential@10": 0.5328292310353233, - "precision@1": 0.38, + "hit@10": 0.7, + "ndcg_linear@1": 0.382, + "ndcg_linear@5": 0.5094295946941956, + "ndcg_linear@10": 0.5343016614660261, + "ndcg_exponential@1": 0.382, + "ndcg_exponential@5": 0.5094295946941956, + "ndcg_exponential@10": 0.5343016614660261, + "precision@1": 0.382, "precision@5": 0.12459999999999999, - "precision@10": 0.06969999999999998, - "recall@1": 0.38, + "precision@10": 0.07, + "recall@1": 0.382, "recall@5": 0.623, - "recall@10": 0.697, - "f1@1": 0.38, + "recall@10": 0.7, + "f1@1": 0.382, "f1@5": 0.20766666666666675, - "f1@10": 0.12672727272727274, - "map@1": 0.38, - "map@5": 0.4706833333333333, - "map@10": 0.4807186507936508, - "mrr@1": 0.38, - "mrr@5": 0.4706833333333333, - "mrr@10": 0.4807186507936508, + "f1@10": 0.12727272727272732, + "map@1": 0.382, + "map@5": 0.47164999999999996, + "map@10": 0.481895634920635, + "mrr@1": 0.382, + "mrr@5": 0.47164999999999996, + "mrr@10": 0.481895634920635, "num_pred": 1000, "num_data": 1000 }, "SmthSmthV2": { - "hit@1": 0.428, - "hit@5": 0.692, - "hit@10": 0.786, - "ndcg_linear@1": 0.428, - "ndcg_linear@5": 0.5672834391090181, - "ndcg_linear@10": 0.5981163502873708, - "ndcg_exponential@1": 0.428, - "ndcg_exponential@5": 0.5672834391090181, - "ndcg_exponential@10": 0.5981163502873708, - "precision@1": 0.428, - "precision@5": 0.13839999999999997, - "precision@10": 0.07859999999999999, - "recall@1": 0.428, - "recall@5": 0.692, - "recall@10": 0.786, - "f1@1": 0.428, - "f1@5": 0.23066666666666674, - "f1@10": 0.14290909090909093, - "map@1": 0.428, - "map@5": 0.5259333333333333, - "map@10": 0.5389079365079364, - "mrr@1": 0.428, - "mrr@5": 0.5259333333333333, - "mrr@10": 0.5389079365079364, + "hit@1": 0.43, + "hit@5": 0.695, + "hit@10": 0.787, + "ndcg_linear@1": 0.43, + "ndcg_linear@5": 0.5693491743977073, + "ndcg_linear@10": 0.5995018159445703, + "ndcg_exponential@1": 0.43, + "ndcg_exponential@5": 0.5693491743977073, + "ndcg_exponential@10": 0.5995018159445703, + "precision@1": 0.43, + "precision@5": 0.139, + "precision@10": 0.0787, + "recall@1": 0.43, + "recall@5": 0.695, + "recall@10": 0.787, + "f1@1": 0.43, + "f1@5": 0.23166666666666674, + "f1@10": 0.14309090909090913, + "map@1": 0.43, + "map@5": 0.52775, + "map@10": 0.5404253968253968, + "mrr@1": 0.43, + "mrr@5": 0.52775, + "mrr@10": 0.5404253968253968, "num_pred": 1000, "num_data": 1000 }, "HMDB51": { - "hit@1": 0.409, - "hit@5": 0.638, - "hit@10": 0.713, - "ndcg_linear@1": 0.409, - "ndcg_linear@5": 0.5335348498920724, - "ndcg_linear@10": 0.5576824580140078, - "ndcg_exponential@1": 0.409, - "ndcg_exponential@5": 0.5335348498920724, - "ndcg_exponential@10": 0.5576824580140078, - "precision@1": 0.409, - "precision@5": 0.12760000000000002, - "precision@10": 0.0713, - "recall@1": 0.409, - "recall@5": 0.638, - "recall@10": 0.713, - "f1@1": 0.409, - "f1@5": 0.21266666666666673, - "f1@10": 0.12963636363636366, - "map@1": 0.409, - "map@5": 0.49854999999999994, - "map@10": 0.5084662698412699, - "mrr@1": 0.409, - "mrr@5": 0.49854999999999994, - "mrr@10": 0.5084662698412699, + "hit@1": 0.402, + "hit@5": 0.636, + "hit@10": 0.711, + "ndcg_linear@1": 0.402, + "ndcg_linear@5": 0.5302469759945302, + "ndcg_linear@10": 0.5544901075502069, + "ndcg_exponential@1": 0.402, + "ndcg_exponential@5": 0.5302469759945302, + "ndcg_exponential@10": 0.5544901075502069, + "precision@1": 0.402, + "precision@5": 0.12719999999999998, + "precision@10": 0.0711, + "recall@1": 0.402, + "recall@5": 0.636, + "recall@10": 0.711, + "f1@1": 0.402, + "f1@5": 0.21200000000000005, + "f1@10": 0.12927272727272732, + "map@1": 0.402, + "map@5": 0.49473333333333336, + "map@10": 0.5047373015873016, + "mrr@1": 0.402, + "mrr@5": 0.49473333333333336, + "mrr@10": 0.5047373015873016, "num_pred": 1000, "num_data": 1000 }, "UCF101": { "hit@1": 0.6, - "hit@5": 0.831, - "hit@10": 0.888, + "hit@5": 0.835, + "hit@10": 0.886, "ndcg_linear@1": 0.6, - "ndcg_linear@5": 0.728875610371988, - "ndcg_linear@10": 0.7473187422682429, + "ndcg_linear@5": 0.729968626028567, + "ndcg_linear@10": 0.7463092612040272, "ndcg_exponential@1": 0.6, - "ndcg_exponential@5": 0.728875610371988, - "ndcg_exponential@10": 0.7473187422682429, + "ndcg_exponential@5": 0.729968626028567, + "ndcg_exponential@10": 0.7463092612040272, "precision@1": 0.6, - "precision@5": 0.16620000000000001, - "precision@10": 0.08880000000000002, + "precision@5": 0.167, + "precision@10": 0.08860000000000001, "recall@1": 0.6, - "recall@5": 0.831, - "recall@10": 0.888, + "recall@5": 0.835, + "recall@10": 0.886, "f1@1": 0.6, - "f1@5": 0.277, - "f1@10": 0.16145454545454543, + "f1@5": 0.2783333333333334, + "f1@10": 0.1610909090909091, "map@1": 0.6, - "map@5": 0.6944166666666668, - "map@10": 0.7020325396825396, + "map@5": 0.6946333333333333, + "map@10": 0.7012869047619047, "mrr@1": 0.6, - "mrr@5": 0.6944166666666668, - "mrr@10": 0.7020325396825396, + "mrr@5": 0.6946333333333333, + "mrr@10": 0.7012869047619047, "num_pred": 1000, "num_data": 1000 }, "Breakfast": { "hit@1": 0.14780600461893764, - "hit@5": 0.6327944572748267, + "hit@5": 0.648960739030023, "hit@10": 1.0, "ndcg_linear@1": 0.14780600461893764, - "ndcg_linear@5": 0.3858298763211173, - "ndcg_linear@10": 0.5069598731135728, + "ndcg_linear@5": 0.39204277953089967, + "ndcg_linear@10": 0.5074063008379464, "ndcg_exponential@1": 0.14780600461893764, - "ndcg_exponential@5": 0.3858298763211173, - "ndcg_exponential@10": 0.5069598731135728, + "ndcg_exponential@5": 0.39204277953089967, + "ndcg_exponential@10": 0.5074063008379464, "precision@1": 0.14780600461893764, - "precision@5": 0.12655889145496535, + "precision@5": 0.12979214780600462, "precision@10": 0.1, "recall@1": 0.14780600461893764, - "recall@5": 0.6327944572748267, + "recall@5": 0.648960739030023, "recall@10": 1.0, "f1@1": 0.14780600461893764, - "f1@5": 0.21093148575827564, + "f1@5": 0.21632024634334107, "f1@10": 0.1818181818181818, "map@1": 0.14780600461893764, - "map@5": 0.30546574287913775, - "map@10": 0.3568477583489131, + "map@5": 0.30862201693610475, + "map@10": 0.35730048755452914, "mrr@1": 0.14780600461893764, - "mrr@5": 0.30546574287913775, - "mrr@10": 0.3568477583489131, + "mrr@5": 0.30862201693610475, + "mrr@10": 0.35730048755452914, "num_pred": 433, "num_data": 433 }, "MVBench": { - "hit@1": 0.337, + "hit@1": 0.33625, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.337, - "ndcg_linear@5": 0.7011998180757394, - "ndcg_linear@10": 0.7011998180757394, - "ndcg_exponential@1": 0.337, - "ndcg_exponential@5": 0.7011998180757394, - "ndcg_exponential@10": 0.7011998180757394, - "precision@1": 0.337, + "ndcg_linear@1": 0.33625, + "ndcg_linear@5": 0.7008998966827248, + "ndcg_linear@10": 0.7008998966827248, + "ndcg_exponential@1": 0.33625, + "ndcg_exponential@5": 0.7008998966827248, + "ndcg_exponential@10": 0.7008998966827248, + "precision@1": 0.33625, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.337, + "recall@1": 0.33625, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.337, + "f1@1": 0.33625, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.337, - "map@5": 0.6002625, - "map@10": 0.6002625, - "mrr@1": 0.337, - "mrr@5": 0.6002625, - "mrr@10": 0.6002625, + "map@1": 0.33625, + "map@5": 0.5998666666666667, + "map@10": 0.5998666666666667, + "mrr@1": 0.33625, + "mrr@5": 0.5998666666666667, + "mrr@10": 0.5998666666666667, "num_pred": 4000, "num_data": 4000 }, "Video-MME": { - "hit@1": 0.30703703703703705, + "hit@1": 0.30814814814814817, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.30703703703703705, - "ndcg_linear@5": 0.6745941548531916, - "ndcg_linear@10": 0.6745941548531916, - "ndcg_exponential@1": 0.30703703703703705, - "ndcg_exponential@5": 0.6745941548531916, - "ndcg_exponential@10": 0.6745941548531916, - "precision@1": 0.30703703703703705, + "ndcg_linear@1": 0.30814814814814817, + "ndcg_linear@5": 0.6744422818449661, + "ndcg_linear@10": 0.6744422818449661, + "ndcg_exponential@1": 0.30814814814814817, + "ndcg_exponential@5": 0.6744422818449661, + "ndcg_exponential@10": 0.6744422818449661, + "precision@1": 0.30814814814814817, "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, - "recall@1": 0.30703703703703705, + "recall@1": 0.30814814814814817, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.30703703703703705, + "f1@1": 0.30814814814814817, "f1@5": 0.3333333333333334, "f1@10": 0.18181818181818188, - "map@1": 0.30703703703703705, - "map@5": 0.5658024691358025, - "map@10": 0.5658024691358025, - "mrr@1": 0.30703703703703705, - "mrr@5": 0.5658024691358025, - "mrr@10": 0.5658024691358025, + "map@1": 0.30814814814814817, + "map@5": 0.5656481481481481, + "map@10": 0.5656481481481481, + "mrr@1": 0.30814814814814817, + "mrr@5": 0.5656481481481481, + "mrr@10": 0.5656481481481481, "num_pred": 2700, "num_data": 2700 }, "NExTQA": { - "hit@1": 0.20924801494628678, + "hit@1": 0.2087809434843531, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.20924801494628678, - "ndcg_linear@5": 0.6098815656327321, - "ndcg_linear@10": 0.6098815656327321, - "ndcg_exponential@1": 0.20924801494628678, - "ndcg_exponential@5": 0.6098815656327321, - "ndcg_exponential@10": 0.6098815656327321, - "precision@1": 0.20924801494628678, + "ndcg_linear@1": 0.2087809434843531, + "ndcg_linear@5": 0.6095940067581891, + "ndcg_linear@10": 0.6095940067581891, + "ndcg_exponential@1": 0.2087809434843531, + "ndcg_exponential@5": 0.6095940067581891, + "ndcg_exponential@10": 0.6095940067581891, + "precision@1": 0.2087809434843531, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.20924801494628678, + "recall@1": 0.2087809434843531, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.20924801494628678, + "f1@1": 0.2087809434843531, "f1@5": 0.3333333333333333, "f1@10": 0.1818181818181818, - "map@1": 0.20924801494628678, - "map@5": 0.4819554725206289, - "map@10": 0.4819554725206289, - "mrr@1": 0.20924801494628678, - "mrr@5": 0.4817803207224038, - "mrr@10": 0.4817803207224038, + "map@1": 0.2087809434843531, + "map@5": 0.4815779230888993, + "map@10": 0.4815779230888993, + "mrr@1": 0.2087809434843531, + "mrr@5": 0.48140277129067416, + "mrr@10": 0.48140277129067416, "num_pred": 8564, "num_data": 8564 }, "EgoSchema": { - "hit@1": 0.34, + "hit@1": 0.35, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.34, - "ndcg_linear@5": 0.6804792280158065, - "ndcg_linear@10": 0.6804792280158065, - "ndcg_exponential@1": 0.34, - "ndcg_exponential@5": 0.6804792280158065, - "ndcg_exponential@10": 0.6804792280158065, - "precision@1": 0.34, + "ndcg_linear@1": 0.35, + "ndcg_linear@5": 0.6846570013748754, + "ndcg_linear@10": 0.6846570013748754, + "ndcg_exponential@1": 0.35, + "ndcg_exponential@5": 0.6846570013748754, + "ndcg_exponential@10": 0.6846570013748754, + "precision@1": 0.35, "precision@5": 0.2, "precision@10": 0.1, - "recall@1": 0.34, + "recall@1": 0.35, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.34, + "f1@1": 0.35, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.34, - "map@5": 0.5750333333333333, - "map@10": 0.5750333333333333, - "mrr@1": 0.34, - "mrr@5": 0.5750333333333333, - "mrr@10": 0.5750333333333333, + "map@1": 0.35, + "map@5": 0.5806666666666667, + "map@10": 0.5806666666666667, + "mrr@1": 0.35, + "mrr@5": 0.5806666666666667, + "mrr@10": 0.5806666666666667, "num_pred": 500, "num_data": 500 }, "ActivityNetQA": { - "hit@1": 0.523, + "hit@1": 0.53, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.523, - "ndcg_linear@5": 0.8239534924535853, - "ndcg_linear@10": 0.8239534924535853, - "ndcg_exponential@1": 0.523, - "ndcg_exponential@5": 0.8239534924535853, - "ndcg_exponential@10": 0.8239534924535853, - "precision@1": 0.523, + "ndcg_linear@1": 0.53, + "ndcg_linear@5": 0.8265369841785851, + "ndcg_linear@10": 0.8265369841785851, + "ndcg_exponential@1": 0.53, + "ndcg_exponential@5": 0.8265369841785851, + "ndcg_exponential@10": 0.8265369841785851, + "precision@1": 0.53, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.523, + "recall@1": 0.53, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.523, + "f1@1": 0.53, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.523, - "map@5": 0.7615, - "map@10": 0.7615, - "mrr@1": 0.523, - "mrr@5": 0.7615, - "mrr@10": 0.7615, + "map@1": 0.53, + "map@5": 0.765, + "map@10": 0.765, + "mrr@1": 0.53, + "mrr@5": 0.765, + "mrr@10": 0.765, "num_pred": 1000, "num_data": 1000 }, "DiDeMo": { - "hit@1": 0.30378486055776893, + "hit@1": 0.299800796812749, "hit@5": 0.5388446215139442, - "hit@10": 0.6314741035856574, - "ndcg_linear@1": 0.30378486055776893, - "ndcg_linear@5": 0.42742924340418204, - "ndcg_linear@10": 0.4573351329841821, - "ndcg_exponential@1": 0.30378486055776893, - "ndcg_exponential@5": 0.42742924340418204, - "ndcg_exponential@10": 0.4573351329841821, - "precision@1": 0.30378486055776893, + "hit@10": 0.6334661354581673, + "ndcg_linear@1": 0.299800796812749, + "ndcg_linear@5": 0.42604614232446447, + "ndcg_linear@10": 0.45651350742150193, + "ndcg_exponential@1": 0.299800796812749, + "ndcg_exponential@5": 0.42604614232446447, + "ndcg_exponential@10": 0.45651350742150193, + "precision@1": 0.299800796812749, "precision@5": 0.10776892430278884, - "precision@10": 0.06314741035856573, - "recall@1": 0.30378486055776893, + "precision@10": 0.06334661354581672, + "recall@1": 0.299800796812749, "recall@5": 0.5388446215139442, - "recall@10": 0.6314741035856574, - "f1@1": 0.30378486055776893, + "recall@10": 0.6334661354581673, + "f1@1": 0.299800796812749, "f1@5": 0.17961487383798144, - "f1@10": 0.11481347337921044, - "map@1": 0.30378486055776893, - "map@5": 0.39047144754316065, - "map@10": 0.40278568266616077, - "mrr@1": 0.30378486055776893, - "mrr@5": 0.39047144754316065, - "mrr@10": 0.40278568266616077, + "f1@10": 0.11517566099239408, + "map@1": 0.299800796812749, + "map@5": 0.38857901726427624, + "map@10": 0.4010754600645039, + "mrr@1": 0.299800796812749, + "mrr@5": 0.38857901726427624, + "mrr@10": 0.4010754600645039, "num_pred": 1004, "num_data": 1004 }, "MSR-VTT": { - "hit@1": 0.283, - "hit@5": 0.504, - "hit@10": 0.594, - "ndcg_linear@1": 0.283, - "ndcg_linear@5": 0.3981240408165469, - "ndcg_linear@10": 0.4272757830933265, - "ndcg_exponential@1": 0.283, - "ndcg_exponential@5": 0.3981240408165469, - "ndcg_exponential@10": 0.4272757830933265, - "precision@1": 0.283, - "precision@5": 0.1008, - "precision@10": 0.059399999999999994, - "recall@1": 0.283, - "recall@5": 0.504, - "recall@10": 0.594, - "f1@1": 0.283, - "f1@5": 0.16800000000000007, - "f1@10": 0.10800000000000003, - "map@1": 0.283, - "map@5": 0.36315000000000003, - "map@10": 0.3752111111111111, - "mrr@1": 0.283, - "mrr@5": 0.36315000000000003, - "mrr@10": 0.3752111111111111, + "hit@1": 0.278, + "hit@5": 0.494, + "hit@10": 0.591, + "ndcg_linear@1": 0.278, + "ndcg_linear@5": 0.39215656063414267, + "ndcg_linear@10": 0.4237750890495372, + "ndcg_exponential@1": 0.278, + "ndcg_exponential@5": 0.39215656063414267, + "ndcg_exponential@10": 0.4237750890495372, + "precision@1": 0.278, + "precision@5": 0.09879999999999999, + "precision@10": 0.05909999999999999, + "recall@1": 0.278, + "recall@5": 0.494, + "recall@10": 0.591, + "f1@1": 0.278, + "f1@5": 0.1646666666666667, + "f1@10": 0.10745454545454547, + "map@1": 0.278, + "map@5": 0.3583, + "map@10": 0.37149404761904764, + "mrr@1": 0.278, + "mrr@5": 0.3583, + "mrr@10": 0.37149404761904764, "num_pred": 1000, "num_data": 1000 }, "MSVD": { - "hit@1": 0.48059701492537316, - "hit@5": 0.7597014925373134, - "hit@10": 0.844776119402985, - "ndcg_linear@1": 0.48059701492537316, - "ndcg_linear@5": 0.63096871065978, - "ndcg_linear@10": 0.6588553080260654, - "ndcg_exponential@1": 0.48059701492537316, - "ndcg_exponential@5": 0.63096871065978, - "ndcg_exponential@10": 0.6588553080260654, - "precision@1": 0.48059701492537316, - "precision@5": 0.15194029850746266, - "precision@10": 0.0844776119402985, - "recall@1": 0.48059701492537316, - "recall@5": 0.7597014925373134, - "recall@10": 0.844776119402985, - "f1@1": 0.48059701492537316, - "f1@5": 0.2532338308457712, - "f1@10": 0.15359565807327005, - "map@1": 0.48059701492537316, - "map@5": 0.5879104477611939, - "map@10": 0.5996434494195687, - "mrr@1": 0.48059701492537316, - "mrr@5": 0.5879104477611939, - "mrr@10": 0.5996434494195687, + "hit@1": 0.47313432835820896, + "hit@5": 0.7701492537313432, + "hit@10": 0.835820895522388, + "ndcg_linear@1": 0.47313432835820896, + "ndcg_linear@5": 0.6330185348434713, + "ndcg_linear@10": 0.6546238014476939, + "ndcg_exponential@1": 0.47313432835820896, + "ndcg_exponential@5": 0.6330185348434713, + "ndcg_exponential@10": 0.6546238014476939, + "precision@1": 0.47313432835820896, + "precision@5": 0.15402985074626863, + "precision@10": 0.08358208955223878, + "recall@1": 0.47313432835820896, + "recall@5": 0.7701492537313432, + "recall@10": 0.835820895522388, + "f1@1": 0.47313432835820896, + "f1@5": 0.25671641791044786, + "f1@10": 0.15196743554952513, + "map@1": 0.47313432835820896, + "map@5": 0.5871890547263681, + "map@10": 0.5963190002369106, + "mrr@1": 0.47313432835820896, + "mrr@5": 0.5871890547263681, + "mrr@10": 0.5963190002369106, "num_pred": 670, "num_data": 670 }, "VATEX": { - "hit@1": 0.26462706565430993, - "hit@5": 0.5015631978561857, + "hit@1": 0.2615006699419384, + "hit@5": 0.5011165698972756, "hit@10": 0.6045109423849933, - "ndcg_linear@1": 0.26462706565430993, - "ndcg_linear@5": 0.3881950217969347, - "ndcg_linear@10": 0.4215820887400626, - "ndcg_exponential@1": 0.26462706565430993, - "ndcg_exponential@5": 0.3881950217969347, - "ndcg_exponential@10": 0.4215820887400626, - "precision@1": 0.26462706565430993, - "precision@5": 0.10031263957123716, - "precision@10": 0.06045109423849933, - "recall@1": 0.26462706565430993, - "recall@5": 0.5015631978561857, + "ndcg_linear@1": 0.2615006699419384, + "ndcg_linear@5": 0.387199297010428, + "ndcg_linear@10": 0.42071256609715807, + "ndcg_exponential@1": 0.2615006699419384, + "ndcg_exponential@5": 0.387199297010428, + "ndcg_exponential@10": 0.42071256609715807, + "precision@1": 0.2615006699419384, + "precision@5": 0.10022331397945512, + "precision@10": 0.06045109423849934, + "recall@1": 0.2615006699419384, + "recall@5": 0.5011165698972756, "recall@10": 0.6045109423849933, - "f1@1": 0.26462706565430993, - "f1@5": 0.16718773261872863, + "f1@1": 0.2615006699419384, + "f1@5": 0.16703885663242524, "f1@10": 0.10991108043363516, - "map@1": 0.26462706565430993, - "map@5": 0.35071460473425636, - "map@10": 0.364550678094671, - "mrr@1": 0.26462706565430993, - "mrr@5": 0.35071460473425636, - "mrr@10": 0.364550678094671, + "map@1": 0.2615006699419384, + "map@5": 0.3494566026499925, + "map@10": 0.36333361690664057, + "mrr@1": 0.2615006699419384, + "mrr@5": 0.3494566026499925, + "mrr@10": 0.36333361690664057, "num_pred": 4478, "num_data": 4478 }, "YouCook2": { "hit@1": 0.10632274300094369, - "hit@5": 0.23057565272098143, - "hit@10": 0.30009436929852157, + "hit@5": 0.23277760301981756, + "hit@10": 0.29852154765649574, "ndcg_linear@1": 0.10632274300094369, - "ndcg_linear@5": 0.17033555328162955, - "ndcg_linear@10": 0.19266164698283084, + "ndcg_linear@5": 0.17082880801245628, + "ndcg_linear@10": 0.1919396953980378, "ndcg_exponential@1": 0.10632274300094369, - "ndcg_exponential@5": 0.17033555328162955, - "ndcg_exponential@10": 0.19266164698283084, + "ndcg_exponential@5": 0.17082880801245628, + "ndcg_exponential@10": 0.1919396953980378, "precision@1": 0.10632274300094369, - "precision@5": 0.04611513054419629, - "precision@10": 0.030009436929852158, + "precision@5": 0.04655552060396351, + "precision@10": 0.029852154765649577, "recall@1": 0.10632274300094369, - "recall@5": 0.23057565272098143, - "recall@10": 0.30009436929852157, + "recall@5": 0.23277760301981756, + "recall@10": 0.29852154765649574, "f1@1": 0.10632274300094369, - "f1@5": 0.07685855090699381, - "f1@10": 0.054562612599731185, + "f1@5": 0.07759253433993919, + "f1@10": 0.05427664502845378, "map@1": 0.10632274300094369, - "map@5": 0.150487574709028, - "map@10": 0.15961231194795208, + "map@5": 0.15048233197022123, + "map@10": 0.1591123793545939, "mrr@1": 0.10632274300094369, - "mrr@5": 0.150487574709028, - "mrr@10": 0.15961231194795208, + "mrr@5": 0.15048233197022123, + "mrr@10": 0.1591123793545939, "num_pred": 3179, "num_data": 3179 }, "QVHighlight": { - "hit@1": 0.4939981532779317, - "hit@5": 0.8679593721144968, + "hit@1": 0.4967682363804247, + "hit@5": 0.8698060941828255, "hit@10": 1.0, - "ndcg_linear@1": 0.4939981532779317, - "ndcg_linear@5": 0.6946135135669718, - "ndcg_linear@10": 0.7373262762832604, - "ndcg_exponential@1": 0.4939981532779317, - "ndcg_exponential@5": 0.6946135135669718, - "ndcg_exponential@10": 0.7373262762832604, - "precision@1": 0.4939981532779317, - "precision@5": 0.17359187442289936, + "ndcg_linear@1": 0.4967682363804247, + "ndcg_linear@5": 0.6963196073180201, + "ndcg_linear@10": 0.7382671456409741, + "ndcg_exponential@1": 0.4967682363804247, + "ndcg_exponential@5": 0.6963196073180201, + "ndcg_exponential@10": 0.7382671456409741, + "precision@1": 0.4967682363804247, + "precision@5": 0.17396121883656507, "precision@10": 0.09999999999999998, - "recall@1": 0.4939981532779317, - "recall@5": 0.8679593721144968, + "recall@1": 0.4967682363804247, + "recall@5": 0.8698060941828255, "recall@10": 1.0, - "f1@1": 0.4939981532779317, - "f1@5": 0.2893197907048323, + "f1@1": 0.4967682363804247, + "f1@5": 0.28993536472760856, "f1@10": 0.18181818181818185, - "map@1": 0.4939981532779317, - "map@5": 0.636734379809172, - "map@10": 0.6543771709976696, - "mrr@1": 0.4939981532779317, - "mrr@5": 0.636734379809172, - "mrr@10": 0.6543771709976696, + "map@1": 0.4967682363804247, + "map@5": 0.6384579870729454, + "map@10": 0.6556838001436339, + "mrr@1": 0.4967682363804247, + "mrr@5": 0.6384579870729454, + "mrr@10": 0.6556838001436339, "num_pred": 1083, "num_data": 1083 }, "Charades-STA": { - "hit@1": 0.20220082530949107, + "hit@1": 0.20082530949105915, "hit@5": 0.6011004126547456, "hit@10": 1.0, - "ndcg_linear@1": 0.20220082530949107, - "ndcg_linear@5": 0.3987840994710363, - "ndcg_linear@10": 0.524544158533619, - "ndcg_exponential@1": 0.20220082530949107, - "ndcg_exponential@5": 0.3987840994710363, - "ndcg_exponential@10": 0.524544158533619, - "precision@1": 0.20220082530949107, + "ndcg_linear@1": 0.20082530949105915, + "ndcg_linear@5": 0.3987228595401552, + "ndcg_linear@10": 0.5240740044205356, + "ndcg_exponential@1": 0.20082530949105915, + "ndcg_exponential@5": 0.3987228595401552, + "ndcg_exponential@10": 0.5240740044205356, + "precision@1": 0.20082530949105915, "precision@5": 0.1202200825309491, "precision@10": 0.09999999999999998, - "recall@1": 0.20220082530949107, + "recall@1": 0.20082530949105915, "recall@5": 0.6011004126547456, "recall@10": 1.0, - "f1@1": 0.20220082530949107, + "f1@1": 0.20082530949105915, "f1@5": 0.20036680421824854, "f1@10": 0.18181818181818185, - "map@1": 0.20220082530949107, - "map@5": 0.3330353049060064, - "map@10": 0.38307897207484554, - "mrr@1": 0.20220082530949107, - "mrr@5": 0.3330353049060064, - "mrr@10": 0.38307897207484554, + "map@1": 0.20082530949105915, + "map@5": 0.3328748280605227, + "map@10": 0.38251020720071616, + "mrr@1": 0.20082530949105915, + "mrr@5": 0.3328748280605227, + "mrr@10": 0.38251020720071616, "num_pred": 727, "num_data": 727 }, "MomentSeeker": { + "hit@1": 0.4288389513108614, + "hit@5": 0.8995006242197253, + "hit@10": 1.0, + "ndcg_linear@1": 0.4288389513108614, + "ndcg_linear@5": 0.6773380464698513, + "ndcg_linear@10": 0.7173483115377756, + "ndcg_exponential@1": 0.4288389513108614, + "ndcg_exponential@5": 0.6773380464698513, + "ndcg_exponential@10": 0.7173483115377756, + "precision@1": 0.4288389513108614, + "precision@5": 0.199625468164794, + "precision@10": 0.11429463171036207, + "recall@1": 0.4018154390345402, + "recall@5": 0.8818404078235539, + "recall@10": 0.9979972950478567, + "f1@1": 0.41009947882607056, + "f1@5": 0.31994734007842623, + "f1@10": 0.20223136725945715, + "map@1": 0.4288389513108614, + "map@5": 0.6024493688444998, + "map@10": 0.6217935975368091, + "mrr@1": 0.4288389513108614, + "mrr@5": 0.6147940074906366, + "mrr@10": 0.6290029328418841, + "num_pred": 1602, + "num_data": 1602 + }, + "MomentSeeker_1k8": { "hit@1": 0.4083333333333333, "hit@5": 0.8861111111111111, "hit@10": 1.0, @@ -1525,422 +1553,422 @@ }, "visdoc": { "ViDoRe_arxivqa": { - "hit@1": 0.734, - "hit@5": 0.866, - "hit@10": 0.906, - "ndcg_linear@1": 0.734, - "ndcg_linear@5": 0.8057573987022552, - "ndcg_linear@10": 0.8188064742981096, - "ndcg_exponential@1": 0.734, - "ndcg_exponential@5": 0.8057573987022552, - "ndcg_exponential@10": 0.8188064742981096, - "precision@1": 0.734, - "precision@5": 0.1732, - "precision@10": 0.0906, - "recall@1": 0.734, - "recall@5": 0.866, - "recall@10": 0.906, - "f1@1": 0.734, - "f1@5": 0.28866666666666674, - "f1@10": 0.16472727272727275, - "map@1": 0.734, - "map@5": 0.7856000000000001, - "map@10": 0.7910603174603175, - "mrr@1": 0.734, - "mrr@5": 0.7856000000000001, - "mrr@10": 0.7910603174603175, + "hit@1": 0.724, + "hit@5": 0.842, + "hit@10": 0.888, + "ndcg_linear@1": 0.724, + "ndcg_linear@5": 0.7887239995764236, + "ndcg_linear@10": 0.8037822405354991, + "ndcg_exponential@1": 0.724, + "ndcg_exponential@5": 0.7887239995764236, + "ndcg_exponential@10": 0.8037822405354991, + "precision@1": 0.724, + "precision@5": 0.1684, + "precision@10": 0.0888, + "recall@1": 0.724, + "recall@5": 0.842, + "recall@10": 0.888, + "f1@1": 0.724, + "f1@5": 0.2806666666666667, + "f1@10": 0.16145454545454543, + "map@1": 0.724, + "map@5": 0.7707999999999999, + "map@10": 0.7771166666666666, + "mrr@1": 0.724, + "mrr@5": 0.7707999999999999, + "mrr@10": 0.7771166666666666, "num_pred": 500, "num_data": 500 }, "ViDoRe_docvqa": { - "hit@1": 0.35033259423503327, - "hit@5": 0.541019955654102, - "hit@10": 0.6208425720620843, - "ndcg_linear@1": 0.35033259423503327, - "ndcg_linear@5": 0.4484886950768515, - "ndcg_linear@10": 0.47338485947788617, - "ndcg_exponential@1": 0.35033259423503327, - "ndcg_exponential@5": 0.4484886950768515, - "ndcg_exponential@10": 0.47338485947788617, - "precision@1": 0.35033259423503327, - "precision@5": 0.10953436807095342, - "precision@10": 0.06341463414634146, - "recall@1": 0.34320029563932003, - "recall@5": 0.5335920177383592, - "recall@10": 0.6106060606060606, - "f1@1": 0.34476093012678377, - "f1@5": 0.18059028489444562, - "f1@10": 0.11417931661834103, - "map@1": 0.35033259423503327, - "map@5": 0.41779995072678, - "map@10": 0.42790588814979064, - "mrr@1": 0.35033259423503327, - "mrr@5": 0.4246119733924612, - "mrr@10": 0.4349144757681343, + "hit@1": 0.2926829268292683, + "hit@5": 0.4523281596452328, + "hit@10": 0.5210643015521065, + "ndcg_linear@1": 0.2926829268292683, + "ndcg_linear@5": 0.3706351906832084, + "ndcg_linear@10": 0.39380824105451623, + "ndcg_exponential@1": 0.2926829268292683, + "ndcg_exponential@5": 0.3706351906832084, + "ndcg_exponential@10": 0.39380824105451623, + "precision@1": 0.2926829268292683, + "precision@5": 0.09046563192904655, + "precision@10": 0.05321507760532151, + "recall@1": 0.2889874353288987, + "recall@5": 0.44493717664449367, + "recall@10": 0.5155949741315595, + "f1@1": 0.29009608277900956, + "f1@5": 0.14998416217928415, + "f1@10": 0.09613032938975291, + "map@1": 0.2926829268292683, + "map@5": 0.34457378664695737, + "map@10": 0.35440555379579775, + "mrr@1": 0.2926829268292683, + "mrr@5": 0.3495934959349593, + "mrr@10": 0.35850666948227927, "num_pred": 451, "num_data": 451 }, "ViDoRe_infovqa": { - "hit@1": 0.7732793522267206, - "hit@5": 0.8947368421052632, - "hit@10": 0.9412955465587044, - "ndcg_linear@1": 0.7732793522267206, - "ndcg_linear@5": 0.836933803347616, - "ndcg_linear@10": 0.852452939253696, - "ndcg_exponential@1": 0.7732793522267206, - "ndcg_exponential@5": 0.836933803347616, - "ndcg_exponential@10": 0.852452939253696, - "precision@1": 0.7732793522267206, - "precision@5": 0.17894736842105263, - "precision@10": 0.09433198380566803, - "recall@1": 0.7732793522267206, - "recall@5": 0.8937246963562753, - "recall@10": 0.9412955465587044, - "f1@1": 0.7732793522267206, - "f1@5": 0.29814921920185083, - "f1@10": 0.17145135566188197, - "map@1": 0.7732793522267206, - "map@5": 0.817914979757085, - "map@10": 0.8244569757727652, - "mrr@1": 0.7732793522267206, - "mrr@5": 0.8182523616734144, - "mrr@10": 0.8245051731893837, + "hit@1": 0.7388663967611336, + "hit@5": 0.9008097165991903, + "hit@10": 0.9352226720647774, + "ndcg_linear@1": 0.7388663967611336, + "ndcg_linear@5": 0.8270703385192045, + "ndcg_linear@10": 0.8390004324621297, + "ndcg_exponential@1": 0.7388663967611336, + "ndcg_exponential@5": 0.8270703385192045, + "ndcg_exponential@10": 0.8390004324621297, + "precision@1": 0.7388663967611336, + "precision@5": 0.18016194331983806, + "precision@10": 0.0937246963562753, + "recall@1": 0.7388663967611336, + "recall@5": 0.8981781376518219, + "recall@10": 0.9336032388663967, + "f1@1": 0.7388663967611336, + "f1@5": 0.29990360516676307, + "f1@10": 0.1702490491964176, + "map@1": 0.7388663967611336, + "map@5": 0.8027327935222672, + "map@10": 0.8079879185142343, + "mrr@1": 0.7388663967611336, + "mrr@5": 0.8036437246963563, + "mrr@10": 0.8086458132510763, "num_pred": 494, "num_data": 494 }, "ViDoRe_tabfquad": { - "hit@1": 0.8107142857142857, - "hit@5": 0.9571428571428572, - "hit@10": 0.975, - "ndcg_linear@1": 0.8107142857142857, - "ndcg_linear@5": 0.8920767921191974, - "ndcg_linear@10": 0.8982104341600656, - "ndcg_exponential@1": 0.8107142857142857, - "ndcg_exponential@5": 0.8920767921191974, - "ndcg_exponential@10": 0.8982104341600656, - "precision@1": 0.8107142857142857, - "precision@5": 0.19142857142857142, - "precision@10": 0.09749999999999999, - "recall@1": 0.8107142857142857, - "recall@5": 0.9571428571428572, - "recall@10": 0.975, - "f1@1": 0.8107142857142857, - "f1@5": 0.31904761904761914, - "f1@10": 0.1772727272727273, - "map@1": 0.8107142857142857, - "map@5": 0.8701190476190476, - "map@10": 0.8728613945578231, - "mrr@1": 0.8107142857142857, - "mrr@5": 0.8701190476190476, - "mrr@10": 0.8728613945578231, + "hit@1": 0.7964285714285714, + "hit@5": 0.9428571428571428, + "hit@10": 0.9642857142857143, + "ndcg_linear@1": 0.7964285714285714, + "ndcg_linear@5": 0.8776640595420117, + "ndcg_linear@10": 0.8847147068994818, + "ndcg_exponential@1": 0.7964285714285714, + "ndcg_exponential@5": 0.8776640595420117, + "ndcg_exponential@10": 0.8847147068994818, + "precision@1": 0.7964285714285714, + "precision@5": 0.18857142857142856, + "precision@10": 0.09642857142857142, + "recall@1": 0.7964285714285714, + "recall@5": 0.9428571428571428, + "recall@10": 0.9642857142857143, + "f1@1": 0.7964285714285714, + "f1@5": 0.3142857142857144, + "f1@10": 0.17532467532467538, + "map@1": 0.7964285714285714, + "map@5": 0.8555952380952381, + "map@10": 0.8585813492063492, + "mrr@1": 0.7964285714285714, + "mrr@5": 0.8555952380952381, + "mrr@10": 0.8585813492063492, "num_pred": 280, "num_data": 280 }, "ViDoRe_tatdqa": { - "hit@1": 0.30741190765492105, - "hit@5": 0.5577156743620899, - "hit@10": 0.6433778857837181, - "ndcg_linear@1": 0.30741190765492105, - "ndcg_linear@5": 0.4382195288905396, - "ndcg_linear@10": 0.4658056006683759, - "ndcg_exponential@1": 0.30741190765492105, - "ndcg_exponential@5": 0.4382195288905396, - "ndcg_exponential@10": 0.4658056006683759, - "precision@1": 0.30741190765492105, - "precision@5": 0.1117861482381531, - "precision@10": 0.06452004860267314, - "recall@1": 0.30650060753341435, - "recall@5": 0.5565613608748481, - "recall@10": 0.6422235722964762, - "f1@1": 0.3068043742405832, - "f1@5": 0.18601515940519592, - "f1@10": 0.11718583158437353, - "map@1": 0.30741190765492105, - "map@5": 0.39878898339408664, - "map@10": 0.41010757198788794, - "mrr@1": 0.30741190765492105, - "mrr@5": 0.39935196435803966, - "mrr@10": 0.41067257806322205, + "hit@1": 0.3128797083839611, + "hit@5": 0.5607533414337789, + "hit@10": 0.6579586877278251, + "ndcg_linear@1": 0.3128797083839611, + "ndcg_linear@5": 0.44290381850341687, + "ndcg_linear@10": 0.47383178418388844, + "ndcg_exponential@1": 0.3128797083839611, + "ndcg_exponential@5": 0.44290381850341687, + "ndcg_exponential@10": 0.47383178418388844, + "precision@1": 0.3128797083839611, + "precision@5": 0.11227217496962331, + "precision@10": 0.06585662211421628, + "recall@1": 0.3125759416767922, + "recall@5": 0.5607533414337789, + "recall@10": 0.6568043742405831, + "f1@1": 0.3126771972458485, + "f1@5": 0.18706243129086395, + "f1@10": 0.11965278544865425, + "map@1": 0.3128797083839611, + "map@5": 0.40380720939651676, + "map@10": 0.41641907269185524, + "mrr@1": 0.3128797083839611, + "mrr@5": 0.4039590927501012, + "mrr@10": 0.41675610908599975, "num_pred": 1646, "num_data": 1646 }, "ViDoRe_shiftproject": { "hit@1": 0.45, - "hit@5": 0.74, - "hit@10": 0.87, + "hit@5": 0.75, + "hit@10": 0.88, "ndcg_linear@1": 0.45, - "ndcg_linear@5": 0.608217112664813, - "ndcg_linear@10": 0.650000125092289, + "ndcg_linear@5": 0.610415276995873, + "ndcg_linear@10": 0.6512285295565106, "ndcg_exponential@1": 0.45, - "ndcg_exponential@5": 0.608217112664813, - "ndcg_exponential@10": 0.650000125092289, + "ndcg_exponential@5": 0.610415276995873, + "ndcg_exponential@10": 0.6512285295565106, "precision@1": 0.45, - "precision@5": 0.14799999999999996, - "precision@10": 0.08699999999999998, + "precision@5": 0.14999999999999997, + "precision@10": 0.08799999999999997, "recall@1": 0.45, - "recall@5": 0.74, - "recall@10": 0.87, + "recall@5": 0.75, + "recall@10": 0.88, "f1@1": 0.45, - "f1@5": 0.24666666666666667, - "f1@10": 0.1581818181818182, + "f1@5": 0.25000000000000006, + "f1@10": 0.16000000000000003, "map@1": 0.45, - "map@5": 0.564, - "map@10": 0.5810912698412699, + "map@5": 0.5638333333333333, + "map@10": 0.579968253968254, "mrr@1": 0.45, - "mrr@5": 0.564, - "mrr@10": 0.5810912698412699, + "mrr@5": 0.5638333333333333, + "mrr@10": 0.579968253968254, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.76, - "hit@5": 0.97, - "hit@10": 0.98, - "ndcg_linear@1": 0.76, - "ndcg_linear@5": 0.8852555261521673, - "ndcg_linear@10": 0.8882658261088071, - "ndcg_exponential@1": 0.76, - "ndcg_exponential@5": 0.8852555261521673, - "ndcg_exponential@10": 0.8882658261088071, - "precision@1": 0.76, - "precision@5": 0.19399999999999995, - "precision@10": 0.09799999999999998, - "recall@1": 0.76, - "recall@5": 0.97, - "recall@10": 0.98, - "f1@1": 0.76, - "f1@5": 0.3233333333333334, - "f1@10": 0.17818181818181827, - "map@1": 0.76, - "map@5": 0.8558333333333333, - "map@10": 0.8569444444444444, - "mrr@1": 0.76, - "mrr@5": 0.8558333333333333, - "mrr@10": 0.8569444444444444, - "num_pred": 100, - "num_data": 100 - }, - "ViDoRe_syntheticDocQA_energy": { "hit@1": 0.79, - "hit@5": 0.93, - "hit@10": 0.97, + "hit@5": 0.98, + "hit@10": 0.99, "ndcg_linear@1": 0.79, - "ndcg_linear@5": 0.8650828475372231, - "ndcg_linear@10": 0.8780588117396043, + "ndcg_linear@5": 0.8912089044754267, + "ndcg_linear@10": 0.894363553243284, "ndcg_exponential@1": 0.79, - "ndcg_exponential@5": 0.8650828475372231, - "ndcg_exponential@10": 0.8780588117396043, + "ndcg_exponential@5": 0.8912089044754267, + "ndcg_exponential@10": 0.894363553243284, "precision@1": 0.79, - "precision@5": 0.18599999999999994, - "precision@10": 0.09699999999999999, + "precision@5": 0.19599999999999995, + "precision@10": 0.09899999999999999, "recall@1": 0.79, - "recall@5": 0.93, - "recall@10": 0.97, + "recall@5": 0.98, + "recall@10": 0.99, "f1@1": 0.79, - "f1@5": 0.31000000000000005, - "f1@10": 0.17636363636363647, + "f1@5": 0.32666666666666677, + "f1@10": 0.18000000000000008, "map@1": 0.79, - "map@5": 0.8433333333333333, - "map@10": 0.8486904761904762, + "map@5": 0.8615, + "map@10": 0.86275, "mrr@1": 0.79, - "mrr@5": 0.8433333333333333, - "mrr@10": 0.8486904761904762, + "mrr@5": 0.8615, + "mrr@10": 0.86275, + "num_pred": 100, + "num_data": 100 + }, + "ViDoRe_syntheticDocQA_energy": { + "hit@1": 0.8, + "hit@5": 0.92, + "hit@10": 0.95, + "ndcg_linear@1": 0.8, + "ndcg_linear@5": 0.8628970749847314, + "ndcg_linear@10": 0.872504443886848, + "ndcg_exponential@1": 0.8, + "ndcg_exponential@5": 0.8628970749847314, + "ndcg_exponential@10": 0.872504443886848, + "precision@1": 0.8, + "precision@5": 0.18399999999999994, + "precision@10": 0.09499999999999996, + "recall@1": 0.8, + "recall@5": 0.92, + "recall@10": 0.95, + "f1@1": 0.8, + "f1@5": 0.3066666666666667, + "f1@10": 0.1727272727272728, + "map@1": 0.8, + "map@5": 0.8440000000000001, + "map@10": 0.8479166666666665, + "mrr@1": 0.8, + "mrr@5": 0.8440000000000001, + "mrr@10": 0.8479166666666665, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_government_reports": { "hit@1": 0.74, - "hit@5": 0.94, + "hit@5": 0.96, "hit@10": 0.98, "ndcg_linear@1": 0.74, - "ndcg_linear@5": 0.8501367970825706, - "ndcg_linear@10": 0.8630184664460052, + "ndcg_linear@5": 0.855794149969463, + "ndcg_linear@10": 0.8622821320706536, "ndcg_exponential@1": 0.74, - "ndcg_exponential@5": 0.8501367970825706, - "ndcg_exponential@10": 0.8630184664460052, + "ndcg_exponential@5": 0.855794149969463, + "ndcg_exponential@10": 0.8622821320706536, "precision@1": 0.74, - "precision@5": 0.18799999999999997, + "precision@5": 0.19199999999999995, "precision@10": 0.09799999999999999, "recall@1": 0.74, - "recall@5": 0.94, + "recall@5": 0.96, "recall@10": 0.98, "f1@1": 0.74, - "f1@5": 0.3133333333333334, + "f1@5": 0.32000000000000006, "f1@10": 0.17818181818181825, "map@1": 0.74, - "map@5": 0.8198333333333332, - "map@10": 0.825111111111111, + "map@5": 0.8213333333333332, + "map@10": 0.8240119047619047, "mrr@1": 0.74, - "mrr@5": 0.8198333333333332, - "mrr@10": 0.825111111111111, + "mrr@5": 0.8213333333333332, + "mrr@10": 0.8240119047619047, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.84, - "hit@5": 0.98, - "hit@10": 0.99, - "ndcg_linear@1": 0.84, - "ndcg_linear@5": 0.9217065065186136, - "ndcg_linear@10": 0.9252685783896938, - "ndcg_exponential@1": 0.84, - "ndcg_exponential@5": 0.9217065065186136, - "ndcg_exponential@10": 0.9252685783896938, - "precision@1": 0.84, - "precision@5": 0.19599999999999998, - "precision@10": 0.09899999999999999, - "recall@1": 0.84, - "recall@5": 0.98, - "recall@10": 0.99, - "f1@1": 0.84, - "f1@5": 0.32666666666666677, - "f1@10": 0.18000000000000008, - "map@1": 0.84, - "map@5": 0.9016666666666667, - "map@10": 0.9033333333333334, - "mrr@1": 0.84, - "mrr@5": 0.9016666666666667, - "mrr@10": 0.9033333333333334, + "hit@1": 0.82, + "hit@5": 0.97, + "hit@10": 0.98, + "ndcg_linear@1": 0.82, + "ndcg_linear@5": 0.9113276335450234, + "ndcg_linear@10": 0.9148897054161036, + "ndcg_exponential@1": 0.82, + "ndcg_exponential@5": 0.9113276335450234, + "ndcg_exponential@10": 0.9148897054161036, + "precision@1": 0.82, + "precision@5": 0.19399999999999995, + "precision@10": 0.09799999999999999, + "recall@1": 0.82, + "recall@5": 0.97, + "recall@10": 0.98, + "f1@1": 0.82, + "f1@5": 0.3233333333333334, + "f1@10": 0.17818181818181825, + "map@1": 0.82, + "map@5": 0.8908333333333335, + "map@10": 0.8925, + "mrr@1": 0.82, + "mrr@5": 0.8908333333333335, + "mrr@10": 0.8925, "num_pred": 100, "num_data": 100 }, "ViDoRe_esg_reports_human_labeled_v2": { "hit@1": 0.38461538461538464, - "hit@5": 0.6730769230769231, + "hit@5": 0.6923076923076923, "hit@10": 0.8461538461538461, "ndcg_linear@1": 0.375, - "ndcg_linear@5": 0.4555552886561195, - "ndcg_linear@10": 0.511493540486886, + "ndcg_linear@5": 0.45794378192749136, + "ndcg_linear@10": 0.5151518627387331, "ndcg_exponential@1": 0.37179487179487186, - "ndcg_exponential@5": 0.4541859231025376, - "ndcg_exponential@10": 0.5099104815026472, + "ndcg_exponential@5": 0.45657441637390933, + "ndcg_exponential@10": 0.5143427365134853, "precision@1": 0.38461538461538464, - "precision@5": 0.19230769230769226, - "precision@10": 0.13076923076923078, + "precision@5": 0.1961538461538461, + "precision@10": 0.1346153846153846, "recall@1": 0.23621794871794868, - "recall@5": 0.5487179487179487, - "recall@10": 0.7033882783882782, + "recall@5": 0.553525641025641, + "recall@10": 0.7109432234432234, "f1@1": 0.2715483234714004, - "f1@5": 0.25723492194080433, - "f1@10": 0.2026450736292365, + "f1@5": 0.2615084262143086, + "f1@10": 0.20765476981540335, "map@1": 0.38461538461538464, - "map@5": 0.3914850427350427, - "map@10": 0.415758547008547, + "map@5": 0.39084401709401706, + "map@10": 0.4150869963369964, "mrr@1": 0.38461538461538464, - "mrr@5": 0.491025641025641, - "mrr@10": 0.5144993894993896, + "mrr@5": 0.4964743589743589, + "mrr@10": 0.5166132478632478, "num_pred": 52, "num_data": 52 }, "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.36875, - "hit@5": 0.7171875, - "hit@10": 0.8203125, - "ndcg_linear@1": 0.36875, - "ndcg_linear@5": 0.4426897295015239, - "ndcg_linear@10": 0.4850950127175607, - "ndcg_exponential@1": 0.36875, - "ndcg_exponential@5": 0.4426897295015239, - "ndcg_exponential@10": 0.4850950127175607, - "precision@1": 0.36875, - "precision@5": 0.203125, - "precision@10": 0.13968750000000002, - "recall@1": 0.23270833333333335, - "recall@5": 0.49864999124649856, - "recall@10": 0.6150380777310923, - "f1@1": 0.2621073457792208, - "f1@5": 0.25601551313270066, - "f1@10": 0.20543311717898116, - "map@1": 0.36875, - "map@5": 0.3719822048611111, - "map@10": 0.3923365021789966, - "mrr@1": 0.36875, - "mrr@5": 0.497265625, - "mrr@10": 0.5111972966269841, + "hit@1": 0.375, + "hit@5": 0.7203125, + "hit@10": 0.8265625, + "ndcg_linear@1": 0.375, + "ndcg_linear@5": 0.4459216014541688, + "ndcg_linear@10": 0.48778748308523917, + "ndcg_exponential@1": 0.375, + "ndcg_exponential@5": 0.4459216014541688, + "ndcg_exponential@10": 0.48778748308523917, + "precision@1": 0.375, + "precision@5": 0.20437500000000003, + "precision@10": 0.14, + "recall@1": 0.23436383928571428, + "recall@5": 0.5042478553921569, + "recall@10": 0.620193651324166, + "f1@1": 0.264611066017316, + "f1@5": 0.2582260109681985, + "f1@10": 0.206419724780938, + "map@1": 0.375, + "map@5": 0.3739782986111111, + "map@10": 0.39368920584726946, + "mrr@1": 0.375, + "mrr@5": 0.5019010416666667, + "mrr@10": 0.5162754216269841, "num_pred": 640, "num_data": 640 }, "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.4525862068965517, + "hit@1": 0.43103448275862066, "hit@5": 0.8103448275862069, - "hit@10": 0.9267241379310345, - "ndcg_linear@1": 0.4525862068965517, - "ndcg_linear@5": 0.42995014841633666, - "ndcg_linear@10": 0.43467280891274934, - "ndcg_exponential@1": 0.4525862068965517, - "ndcg_exponential@5": 0.4299501484163365, - "ndcg_exponential@10": 0.43467280891274934, - "precision@1": 0.4525862068965517, - "precision@5": 0.38448275862068965, - "precision@10": 0.33103448275862074, - "recall@1": 0.049016392011628045, - "recall@5": 0.22829352926193736, - "recall@10": 0.36167397868493684, - "f1@1": 0.08115811949451532, - "f1@5": 0.22694280671921516, - "f1@10": 0.27660068249123476, - "map@1": 0.4525862068965517, - "map@5": 0.3282327586206897, - "map@10": 0.2973695391710324, - "mrr@1": 0.4525862068965517, - "mrr@5": 0.5951867816091955, - "mrr@10": 0.6120929118773947, + "hit@10": 0.9181034482758621, + "ndcg_linear@1": 0.43103448275862066, + "ndcg_linear@5": 0.4233377682889509, + "ndcg_linear@10": 0.4285592549217132, + "ndcg_exponential@1": 0.43103448275862066, + "ndcg_exponential@5": 0.4233377682889508, + "ndcg_exponential@10": 0.4285592549217132, + "precision@1": 0.43103448275862066, + "precision@5": 0.3836206896551725, + "precision@10": 0.3314655172413793, + "recall@1": 0.038763928957440845, + "recall@5": 0.22900225812159283, + "recall@10": 0.36110528401874314, + "f1@1": 0.06800130504459743, + "f1@5": 0.22771366337223792, + "f1@10": 0.27657890798021934, + "map@1": 0.43103448275862066, + "map@5": 0.3216606800766283, + "map@10": 0.2913414828335548, + "mrr@1": 0.43103448275862066, + "mrr@5": 0.5794540229885058, + "mrr@10": 0.5947095648604269, "num_pred": 232, "num_data": 232 }, "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.4692982456140351, - "hit@5": 0.7719298245614035, - "hit@10": 0.8640350877192983, - "ndcg_linear@1": 0.4692982456140351, - "ndcg_linear@5": 0.466179429204456, - "ndcg_linear@10": 0.5017617250757785, - "ndcg_exponential@1": 0.4692982456140351, - "ndcg_exponential@5": 0.466179429204456, - "ndcg_exponential@10": 0.5017617250757785, - "precision@1": 0.4692982456140351, + "hit@1": 0.4473684210526316, + "hit@5": 0.7587719298245614, + "hit@10": 0.8596491228070176, + "ndcg_linear@1": 0.4473684210526316, + "ndcg_linear@5": 0.4573956897779833, + "ndcg_linear@10": 0.49047836814154644, + "ndcg_exponential@1": 0.4473684210526316, + "ndcg_exponential@5": 0.4573956897779833, + "ndcg_exponential@10": 0.49047836814154644, + "precision@1": 0.4473684210526316, "precision@5": 0.21929824561403508, - "precision@10": 0.15131578947368426, - "recall@1": 0.2380325814536341, - "recall@5": 0.4968671679197995, - "recall@10": 0.6020624477861319, - "f1@1": 0.28556649037592774, - "f1@5": 0.26698957296533893, - "f1@10": 0.21278941819922756, - "map@1": 0.4692982456140351, - "map@5": 0.38164961013645227, - "map@10": 0.39600202763854087, - "mrr@1": 0.4692982456140351, - "mrr@5": 0.5818713450292398, - "mrr@10": 0.594714216095795, + "precision@10": 0.15000000000000002, + "recall@1": 0.22064536340852128, + "recall@5": 0.491530910609858, + "recall@10": 0.5931547619047618, + "f1@1": 0.26654983955346934, + "f1@5": 0.2649157609844062, + "f1@10": 0.21027052281329617, + "map@1": 0.4473684210526316, + "map@5": 0.37191764132553606, + "map@10": 0.3836273989371577, + "mrr@1": 0.4473684210526316, + "mrr@5": 0.5687134502923976, + "mrr@10": 0.5824178501810082, "num_pred": 228, "num_data": 228 }, "VisRAG_ArxivQA": { - "hit@1": 0.7022058823529411, - "hit@5": 0.821078431372549, - "hit@10": 0.8615196078431373, - "ndcg_linear@1": 0.7022058823529411, - "ndcg_linear@5": 0.7687028213602233, - "ndcg_linear@10": 0.7817454060088397, - "ndcg_exponential@1": 0.7022058823529411, - "ndcg_exponential@5": 0.7687028213602233, - "ndcg_exponential@10": 0.7817454060088397, - "precision@1": 0.7022058823529411, - "precision@5": 0.1642156862745098, - "precision@10": 0.08615196078431371, - "recall@1": 0.7022058823529411, - "recall@5": 0.821078431372549, - "recall@10": 0.8615196078431373, - "f1@1": 0.7022058823529411, - "f1@5": 0.2736928104575164, - "f1@10": 0.15663992869875223, - "map@1": 0.7022058823529411, - "map@5": 0.7510212418300654, - "map@10": 0.7563827614379085, - "mrr@1": 0.7022058823529411, - "mrr@5": 0.7510212418300654, - "mrr@10": 0.7563827614379085, + "hit@1": 0.6997549019607843, + "hit@5": 0.8198529411764706, + "hit@10": 0.8651960784313726, + "ndcg_linear@1": 0.6997549019607843, + "ndcg_linear@5": 0.7673996510312443, + "ndcg_linear@10": 0.7819324552611429, + "ndcg_exponential@1": 0.6997549019607843, + "ndcg_exponential@5": 0.7673996510312443, + "ndcg_exponential@10": 0.7819324552611429, + "precision@1": 0.6997549019607843, + "precision@5": 0.1639705882352941, + "precision@10": 0.08651960784313725, + "recall@1": 0.6997549019607843, + "recall@5": 0.8198529411764706, + "recall@10": 0.8651960784313726, + "f1@1": 0.6997549019607843, + "f1@5": 0.2732843137254903, + "f1@10": 0.1573083778966132, + "map@1": 0.6997549019607843, + "map@5": 0.7496527777777777, + "map@10": 0.7555720899470899, + "mrr@1": 0.6997549019607843, + "mrr@5": 0.7496527777777777, + "mrr@10": 0.7555720899470899, "num_pred": 816, "num_data": 816 }, @@ -1949,11 +1977,11 @@ "hit@5": 0.9206349206349206, "hit@10": 0.9365079365079365, "ndcg_linear@1": 0.7301587301587301, - "ndcg_linear@5": 0.8441010759070336, - "ndcg_linear@10": 0.849392081198039, + "ndcg_linear@5": 0.8420228258503438, + "ndcg_linear@10": 0.8473138311413491, "ndcg_exponential@1": 0.7301587301587301, - "ndcg_exponential@5": 0.8441010759070336, - "ndcg_exponential@10": 0.849392081198039, + "ndcg_exponential@5": 0.8420228258503438, + "ndcg_exponential@10": 0.8473138311413491, "precision@1": 0.7301587301587301, "precision@5": 0.18412698412698403, "precision@10": 0.0936507936507936, @@ -1964,235 +1992,179 @@ "f1@5": 0.3068783068783068, "f1@10": 0.1702741702741703, "map@1": 0.7301587301587301, - "map@5": 0.8174603174603174, - "map@10": 0.8197278911564626, + "map@5": 0.8148148148148149, + "map@10": 0.8170823885109599, "mrr@1": 0.7301587301587301, - "mrr@5": 0.8174603174603174, - "mrr@10": 0.8197278911564626, + "mrr@5": 0.8148148148148149, + "mrr@10": 0.8170823885109599, "num_pred": 63, "num_data": 63 }, "VisRAG_MP-DocVQA": { - "hit@1": 0.5922165820642978, - "hit@5": 0.8206429780033841, + "hit@1": 0.5871404399323181, + "hit@5": 0.8257191201353637, "hit@10": 0.8663282571912013, - "ndcg_linear@1": 0.5922165820642978, - "ndcg_linear@5": 0.717924714040525, - "ndcg_linear@10": 0.7327402059542735, - "ndcg_exponential@1": 0.5922165820642978, - "ndcg_exponential@5": 0.717924714040525, - "ndcg_exponential@10": 0.7327402059542735, - "precision@1": 0.5922165820642978, - "precision@5": 0.16412859560067683, + "ndcg_linear@1": 0.5871404399323181, + "ndcg_linear@5": 0.717983975669403, + "ndcg_linear@10": 0.7313093849286106, + "ndcg_exponential@1": 0.5871404399323181, + "ndcg_exponential@5": 0.717983975669403, + "ndcg_exponential@10": 0.7313093849286106, + "precision@1": 0.5871404399323181, + "precision@5": 0.16514382402707276, "precision@10": 0.08663282571912012, - "recall@1": 0.5922165820642978, - "recall@5": 0.8206429780033841, + "recall@1": 0.5871404399323181, + "recall@5": 0.8257191201353637, "recall@10": 0.8663282571912013, - "f1@1": 0.5922165820642978, - "f1@5": 0.2735476593344614, + "f1@1": 0.5871404399323181, + "f1@5": 0.275239706711788, "f1@10": 0.15751422858021846, - "map@1": 0.5922165820642978, - "map@5": 0.6833051325437113, - "map@10": 0.6894455187602395, - "mrr@1": 0.5922165820642978, - "mrr@5": 0.6833051325437113, - "mrr@10": 0.6894455187602395, + "map@1": 0.5871404399323181, + "map@5": 0.6817540891144951, + "map@10": 0.6873600032229474, + "mrr@1": 0.5871404399323181, + "mrr@5": 0.6817540891144951, + "mrr@10": 0.6873600032229474, "num_pred": 591, "num_data": 591 }, "VisRAG_SlideVQA": { - "hit@1": 0.8794964028776978, - "hit@5": 0.9748201438848921, - "hit@10": 0.9802158273381295, - "ndcg_linear@1": 0.8794964028776978, - "ndcg_linear@5": 0.9150078016078801, - "ndcg_linear@10": 0.9207838812109942, - "ndcg_exponential@1": 0.8794964028776978, - "ndcg_exponential@5": 0.9150078016078801, - "ndcg_exponential@10": 0.9207838812109942, - "precision@1": 0.8794964028776978, - "precision@5": 0.2384892086330935, - "precision@10": 0.12176258992805757, - "recall@1": 0.7598920863309353, - "recall@5": 0.954136690647482, - "recall@10": 0.9694244604316546, - "f1@1": 0.7997601918465228, - "f1@5": 0.37478588557725256, - "f1@10": 0.2138652714192283, - "map@1": 0.8794964028776978, - "map@5": 0.8905575539568344, - "map@10": 0.894137547105173, - "mrr@1": 0.8794964028776978, - "mrr@5": 0.9226618705035972, - "mrr@10": 0.9233213429256595, + "hit@1": 0.8758992805755396, + "hit@5": 0.9766187050359713, + "hit@10": 0.9784172661870504, + "ndcg_linear@1": 0.8758992805755396, + "ndcg_linear@5": 0.9144435190831224, + "ndcg_linear@10": 0.9190700772421555, + "ndcg_exponential@1": 0.8758992805755396, + "ndcg_exponential@5": 0.9144435190831224, + "ndcg_exponential@10": 0.9190700772421555, + "precision@1": 0.8758992805755396, + "precision@5": 0.23884892086330936, + "precision@10": 0.12158273381294964, + "recall@1": 0.7562949640287769, + "recall@5": 0.9559352517985612, + "recall@10": 0.9676258992805755, + "f1@1": 0.7961630695443646, + "f1@5": 0.37538540596094566, + "f1@10": 0.21353826030085027, + "map@1": 0.8758992805755396, + "map@5": 0.8894784172661869, + "map@10": 0.8926002055498459, + "mrr@1": 0.8758992805755396, + "mrr@5": 0.9209232613908872, + "mrr@10": 0.9211031175059952, "num_pred": 556, "num_data": 556 }, "VisRAG_InfoVQA": { - "hit@1": 0.7465181058495822, - "hit@5": 0.9415041782729805, - "hit@10": 0.9665738161559888, - "ndcg_linear@1": 0.7465181058495822, - "ndcg_linear@5": 0.8565495992280632, - "ndcg_linear@10": 0.8647865140019099, - "ndcg_exponential@1": 0.7465181058495822, - "ndcg_exponential@5": 0.8565495992280632, - "ndcg_exponential@10": 0.8647865140019099, - "precision@1": 0.7465181058495822, - "precision@5": 0.18830083565459607, - "precision@10": 0.09665738161559885, - "recall@1": 0.7465181058495822, - "recall@5": 0.9415041782729805, - "recall@10": 0.9665738161559888, - "f1@1": 0.7465181058495822, - "f1@5": 0.31383472609099355, - "f1@10": 0.1757406938465435, - "map@1": 0.7465181058495822, - "map@5": 0.827669452181987, - "map@10": 0.8311491356059602, - "mrr@1": 0.7465181058495822, - "mrr@5": 0.827669452181987, - "mrr@10": 0.8311491356059602, + "hit@1": 0.7479108635097493, + "hit@5": 0.9428969359331476, + "hit@10": 0.9637883008356546, + "ndcg_linear@1": 0.7479108635097493, + "ndcg_linear@5": 0.8587683213119244, + "ndcg_linear@10": 0.8656019755249378, + "ndcg_exponential@1": 0.7479108635097493, + "ndcg_exponential@5": 0.8587683213119244, + "ndcg_exponential@10": 0.8656019755249378, + "precision@1": 0.7479108635097493, + "precision@5": 0.1885793871866295, + "precision@10": 0.09637883008356544, + "recall@1": 0.7479108635097493, + "recall@5": 0.9428969359331476, + "recall@10": 0.9637883008356546, + "f1@1": 0.7479108635097493, + "f1@5": 0.31429897864438266, + "f1@10": 0.17523423651557363, + "map@1": 0.7479108635097493, + "map@5": 0.8301067780872795, + "map@10": 0.8329768536941239, + "mrr@1": 0.7479108635097493, + "mrr@5": 0.8301067780872795, + "mrr@10": 0.8329768536941239, "num_pred": 718, "num_data": 718 }, "VisRAG_PlotQA": { - "hit@1": 0.5052143684820394, - "hit@5": 0.7937427578215527, - "hit@10": 0.8887601390498262, - "ndcg_linear@1": 0.5052143684820394, - "ndcg_linear@5": 0.6604877574932249, - "ndcg_linear@10": 0.6916336549342924, - "ndcg_exponential@1": 0.5052143684820394, - "ndcg_exponential@5": 0.6604877574932249, - "ndcg_exponential@10": 0.6916336549342924, - "precision@1": 0.5052143684820394, - "precision@5": 0.15874855156431056, - "precision@10": 0.08887601390498262, - "recall@1": 0.5052143684820394, - "recall@5": 0.7937427578215527, - "recall@10": 0.8887601390498262, - "f1@1": 0.5052143684820394, - "f1@5": 0.26458091927385097, - "f1@10": 0.1615927525545139, - "map@1": 0.5052143684820394, - "map@5": 0.6159907300115876, - "map@10": 0.629089646673656, - "mrr@1": 0.5052143684820394, - "mrr@5": 0.6159907300115876, - "mrr@10": 0.629089646673656, + "hit@1": 0.5063731170336037, + "hit@5": 0.7902665121668598, + "hit@10": 0.8910776361529548, + "ndcg_linear@1": 0.5063731170336037, + "ndcg_linear@5": 0.6591464916270031, + "ndcg_linear@10": 0.6921386282792643, + "ndcg_exponential@1": 0.5063731170336037, + "ndcg_exponential@5": 0.6591464916270031, + "ndcg_exponential@10": 0.6921386282792643, + "precision@1": 0.5063731170336037, + "precision@5": 0.15805330243337193, + "precision@10": 0.08910776361529547, + "recall@1": 0.5063731170336037, + "recall@5": 0.7902665121668598, + "recall@10": 0.8910776361529548, + "f1@1": 0.5063731170336037, + "f1@5": 0.2634221707222867, + "f1@10": 0.16201411566417362, + "map@1": 0.5063731170336037, + "map@5": 0.6154113557358052, + "map@10": 0.6292565616435837, + "mrr@1": 0.5063731170336037, + "mrr@5": 0.6154113557358052, + "mrr@10": 0.6292565616435837, "num_pred": 863, "num_data": 863 }, - "ViDoSeek-page": { - "hit@1": 0.03502626970227671, - "hit@5": 0.3975481611208406, - "hit@10": 0.5674255691768827, - "ndcg_linear@1": 0.03502626970227671, - "ndcg_linear@5": 0.21944808482459283, - "ndcg_linear@10": 0.2752564930475152, - "ndcg_exponential@1": 0.03502626970227671, - "ndcg_exponential@5": 0.21944808482459285, - "ndcg_exponential@10": 0.2752564930475153, - "precision@1": 0.03502626970227671, - "precision@5": 0.07950963222416814, - "precision@10": 0.05674255691768828, - "recall@1": 0.03502626970227671, - "recall@5": 0.3975481611208406, - "recall@10": 0.5674255691768827, - "f1@1": 0.03502626970227671, - "f1@5": 0.13251605370694688, - "f1@10": 0.10316828530488775, - "map@1": 0.03502626970227671, - "map@5": 0.16103327495621714, - "map@10": 0.184575236983293, - "mrr@1": 0.03502626970227671, - "mrr@5": 0.16103327495621714, - "mrr@10": 0.184575236983293, - "num_pred": 1142, - "num_data": 1142 - }, - "ViDoSeek-doc": { - "hit@1": 0.9903677758318739, - "hit@5": 0.9973730297723292, - "hit@10": 0.9982486865148862, - "ndcg_linear@1": 0.6762988908347927, - "ndcg_linear@5": 0.8018406693375459, - "ndcg_linear@10": 0.767343136404217, - "ndcg_exponential@1": 0.4519639729797348, - "ndcg_exponential@5": 0.6874729261482034, - "ndcg_exponential@10": 0.6978420362198057, - "precision@1": 0.9903677758318739, - "precision@5": 0.8740805604203151, - "precision@10": 0.7643607705779335, - "recall@1": 0.05845834634908238, - "recall@5": 0.2559977615058973, - "recall@10": 0.4381450224554127, - "f1@1": 0.1096528022965413, - "f1@5": 0.388099043607817, - "f1@10": 0.5446572421004792, - "map@1": 0.9903677758318739, - "map@5": 0.8628546409807355, - "map@10": 0.7436259056064535, - "mrr@1": 0.9903677758318739, - "mrr@5": 0.9936077057793345, - "mrr@10": 0.9937327995996997, + "ViDoSeek-page-fixed": { + "hit@1": 0.6558669001751314, + "hit@5": 0.9229422066549913, + "hit@10": 0.968476357267951, + "ndcg_linear@1": 0.6558669001751314, + "ndcg_linear@5": 0.8029721811670532, + "ndcg_linear@10": 0.8178919635488785, + "ndcg_exponential@1": 0.6558669001751314, + "ndcg_exponential@5": 0.8029721811670532, + "ndcg_exponential@10": 0.8178919635488786, + "precision@1": 0.6558669001751314, + "precision@5": 0.18458844133099822, + "precision@10": 0.0968476357267951, + "recall@1": 0.6558669001751314, + "recall@5": 0.9229422066549913, + "recall@10": 0.968476357267951, + "f1@1": 0.6558669001751314, + "f1@5": 0.3076474022183304, + "f1@10": 0.17608661041235474, + "map@1": 0.6558669001751314, + "map@5": 0.7626240513718622, + "map@10": 0.7688936841520029, + "mrr@1": 0.6558669001751314, + "mrr@5": 0.7626240513718622, + "mrr@10": 0.7688936841520029, "num_pred": 1142, "num_data": 1142 }, - "MMLongBench-page": { - "hit@1": 0.05250596658711217, - "hit@5": 0.24224343675417662, - "hit@10": 0.33890214797136037, - "ndcg_linear@1": 0.05250596658711217, - "ndcg_linear@5": 0.11893466223008214, - "ndcg_linear@10": 0.15008296554980238, - "ndcg_exponential@1": 0.05250596658711217, - "ndcg_exponential@5": 0.11893466223008214, - "ndcg_exponential@10": 0.15008296554980238, - "precision@1": 0.05250596658711217, - "precision@5": 0.056324582338902154, - "precision@10": 0.04307875894988067, - "recall@1": 0.028838504375497214, - "recall@5": 0.1757174110694397, - "recall@10": 0.2610182975338106, - "f1@1": 0.03586377822654672, - "f1@5": 0.08071268195513599, - "f1@10": 0.07038494595369645, - "map@1": 0.05250596658711217, - "map@5": 0.08618237867939538, - "map@10": 0.10006671765099569, - "mrr@1": 0.05250596658711217, - "mrr@5": 0.12171837708830548, - "mrr@10": 0.13489174906239346, - "num_pred": 838, - "num_data": 838 - }, - "MMLongBench-doc": { - "hit@1": 0.6157517899761337, - "hit@5": 0.7482100238663485, - "hit@10": 0.7947494033412887, - "ndcg_linear@1": 0.4287987271280827, - "ndcg_linear@5": 0.43709371712705786, - "ndcg_linear@10": 0.4147091530579679, - "ndcg_exponential@1": 0.2952608250937606, - "ndcg_exponential@5": 0.3633856276662273, - "ndcg_exponential@10": 0.36808988125041325, - "precision@1": 0.6157517899761337, - "precision@5": 0.4797136038186158, - "precision@10": 0.41563245823389017, - "recall@1": 0.01970187328257191, - "recall@5": 0.07488813025872536, - "recall@10": 0.12545835862481708, - "f1@1": 0.03781026144552173, - "f1@5": 0.12459954660237356, - "f1@10": 0.18139093061449052, - "map@1": 0.6157517899761337, - "map@5": 0.44125696101829753, - "map@10": 0.36061324308738785, - "mrr@1": 0.6157517899761337, - "mrr@5": 0.6687947494033414, - "mrr@10": 0.6751861006932606, + "MMLongBench-page-fixed": { + "hit@1": 0.3854415274463007, + "hit@5": 0.5906921241050119, + "hit@10": 0.6551312649164678, + "ndcg_linear@1": 0.3854415274463007, + "ndcg_linear@5": 0.4470434560714668, + "ndcg_linear@10": 0.47274771827925227, + "ndcg_exponential@1": 0.3854415274463007, + "ndcg_exponential@5": 0.4470434560714668, + "ndcg_exponential@10": 0.47274771827925227, + "precision@1": 0.3854415274463007, + "precision@5": 0.14439140811455847, + "precision@10": 0.08663484486873507, + "recall@1": 0.3073687350835323, + "recall@5": 0.5113890783043528, + "recall@10": 0.5802406523468576, + "f1@1": 0.3302429604338912, + "f1@5": 0.2156883334582338, + "f1@10": 0.14463551844038952, + "map@1": 0.3854415274463007, + "map@5": 0.40553235216123046, + "map@10": 0.4181925423974947, + "mrr@1": 0.3854415274463007, + "mrr@5": 0.4606404136833731, + "mrr@10": 0.46937246656817067, "num_pred": 838, "num_data": 838 }