diff --git "a/scores/gme-Qwen2-VL-2B-Instruct.json" "b/scores/gme-Qwen2-VL-2B-Instruct.json" --- "a/scores/gme-Qwen2-VL-2B-Instruct.json" +++ "b/scores/gme-Qwen2-VL-2B-Instruct.json" @@ -1,7 +1,7 @@ { "metadata": { "model_name": "gme-Qwen2-VL-2B-Instruct", - "report_generated_date": "2025-06-28T08:54:18.007017", + "report_generated_date": "2025-12-29T09:27:50.332077", "model_size": 2.21, "url": "https://huggingface.co/Alibaba-NLP/gme-Qwen2-VL-2B-Instruct", "data_source": "TIGER-Lab" @@ -9,758 +9,758 @@ "metrics": { "image": { "ImageNet-1K": { - "hit@1": 0.583, - "hit@5": 0.798, - "hit@10": 0.876, - "ndcg_linear@1": 0.583, - "ndcg_linear@5": 0.6981838939261613, - "ndcg_linear@10": 0.7234976673917299, - "ndcg_exponential@1": 0.583, - "ndcg_exponential@5": 0.6981838939261613, - "ndcg_exponential@10": 0.7234976673917299, - "precision@1": 0.583, - "precision@5": 0.1596, - "precision@10": 0.08760000000000003, - "recall@1": 0.583, - "recall@5": 0.798, - "recall@10": 0.876, - "f1@1": 0.583, - "f1@5": 0.26600000000000007, - "f1@10": 0.1592727272727273, - "map@1": 0.583, - "map@5": 0.6649166666666666, - "map@10": 0.6754202380952381, - "mrr@1": 0.583, - "mrr@5": 0.6649166666666666, - "mrr@10": 0.6754202380952381, + "hit@1": 0.581, + "hit@5": 0.8, + "hit@10": 0.873, + "ndcg_linear@1": 0.581, + "ndcg_linear@5": 0.6985583591271246, + "ndcg_linear@10": 0.7223162954832726, + "ndcg_exponential@1": 0.581, + "ndcg_exponential@5": 0.6985583591271246, + "ndcg_exponential@10": 0.7223162954832726, + "precision@1": 0.581, + "precision@5": 0.16, + "precision@10": 0.08730000000000002, + "recall@1": 0.581, + "recall@5": 0.8, + "recall@10": 0.873, + "f1@1": 0.581, + "f1@5": 0.2666666666666667, + "f1@10": 0.15872727272727272, + "map@1": 0.581, + "map@5": 0.6647333333333334, + "map@10": 0.6746281746031746, + "mrr@1": 0.581, + "mrr@5": 0.6647333333333334, + "mrr@10": 0.6746281746031746, "num_pred": 1000, "num_data": 1000 }, "N24News": { - "hit@1": 0.501, - "hit@5": 0.797, - "hit@10": 0.882, - "ndcg_linear@1": 0.501, - "ndcg_linear@5": 0.6577414489591101, - "ndcg_linear@10": 0.6855990297711865, - "ndcg_exponential@1": 0.501, - "ndcg_exponential@5": 0.6577414489591101, - "ndcg_exponential@10": 0.6855990297711865, - "precision@1": 0.501, - "precision@5": 0.1594, - "precision@10": 0.08820000000000001, - "recall@1": 0.501, - "recall@5": 0.797, - "recall@10": 0.882, - "f1@1": 0.501, - "f1@5": 0.2656666666666667, - "f1@10": 0.16036363636363637, - "map@1": 0.501, - "map@5": 0.6114666666666667, - "map@10": 0.6231809523809524, - "mrr@1": 0.501, - "mrr@5": 0.6114666666666667, - "mrr@10": 0.6231809523809524, + "hit@1": 0.503, + "hit@5": 0.794, + "hit@10": 0.886, + "ndcg_linear@1": 0.503, + "ndcg_linear@5": 0.657503308466253, + "ndcg_linear@10": 0.6875241083557199, + "ndcg_exponential@1": 0.503, + "ndcg_exponential@5": 0.657503308466253, + "ndcg_exponential@10": 0.6875241083557199, + "precision@1": 0.503, + "precision@5": 0.15880000000000002, + "precision@10": 0.08860000000000001, + "recall@1": 0.503, + "recall@5": 0.794, + "recall@10": 0.886, + "f1@1": 0.503, + "f1@5": 0.2646666666666667, + "f1@10": 0.16109090909090906, + "map@1": 0.503, + "map@5": 0.6121333333333333, + "map@10": 0.6246813492063492, + "mrr@1": 0.503, + "mrr@5": 0.6121333333333333, + "mrr@10": 0.6246813492063492, "num_pred": 1000, "num_data": 1000 }, "HatefulMemes": { - "hit@1": 0.525, + "hit@1": 0.537, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.525, - "ndcg_linear@5": 0.8246916329464424, - "ndcg_linear@10": 0.8246916329464424, - "ndcg_exponential@1": 0.525, - "ndcg_exponential@5": 0.8246916329464424, - "ndcg_exponential@10": 0.8246916329464424, - "precision@1": 0.525, + "ndcg_linear@1": 0.537, + "ndcg_linear@5": 0.829120475903585, + "ndcg_linear@10": 0.829120475903585, + "ndcg_exponential@1": 0.537, + "ndcg_exponential@5": 0.829120475903585, + "ndcg_exponential@10": 0.829120475903585, + "precision@1": 0.537, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.525, + "recall@1": 0.537, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.525, + "f1@1": 0.537, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.525, - "map@5": 0.7625, - "map@10": 0.7625, - "mrr@1": 0.525, - "mrr@5": 0.7625, - "mrr@10": 0.7625, + "map@1": 0.537, + "map@5": 0.7685, + "map@10": 0.7685, + "mrr@1": 0.537, + "mrr@5": 0.7685, + "mrr@10": 0.7685, "num_pred": 1000, "num_data": 1000 }, "VOC2007": { - "hit@1": 0.759, - "hit@5": 0.951, - "hit@10": 0.989, - "ndcg_linear@1": 0.759, - "ndcg_linear@5": 0.8652219655533966, - "ndcg_linear@10": 0.8775498310604413, - "ndcg_exponential@1": 0.759, - "ndcg_exponential@5": 0.8652219655533966, - "ndcg_exponential@10": 0.8775498310604413, - "precision@1": 0.759, - "precision@5": 0.19020000000000004, - "precision@10": 0.0989, - "recall@1": 0.759, - "recall@5": 0.951, - "recall@10": 0.989, - "f1@1": 0.759, - "f1@5": 0.317, - "f1@10": 0.17981818181818182, - "map@1": 0.759, - "map@5": 0.8363166666666666, - "map@10": 0.8414277777777777, - "mrr@1": 0.759, - "mrr@5": 0.8363166666666666, - "mrr@10": 0.8414277777777777, + "hit@1": 0.758, + "hit@5": 0.95, + "hit@10": 0.988, + "ndcg_linear@1": 0.758, + "ndcg_linear@5": 0.8644920836897664, + "ndcg_linear@10": 0.876875126388255, + "ndcg_exponential@1": 0.758, + "ndcg_exponential@5": 0.8644920836897664, + "ndcg_exponential@10": 0.876875126388255, + "precision@1": 0.758, + "precision@5": 0.19000000000000003, + "precision@10": 0.09880000000000001, + "recall@1": 0.758, + "recall@5": 0.95, + "recall@10": 0.988, + "f1@1": 0.758, + "f1@5": 0.3166666666666667, + "f1@10": 0.17963636363636362, + "map@1": 0.758, + "map@5": 0.8356333333333333, + "map@10": 0.8408, + "mrr@1": 0.758, + "mrr@5": 0.8356333333333333, + "mrr@10": 0.8408, "num_pred": 1000, "num_data": 1000 }, "SUN397": { - "hit@1": 0.673, - "hit@5": 0.914, - "hit@10": 0.964, - "ndcg_linear@1": 0.673, - "ndcg_linear@5": 0.8099367287839547, - "ndcg_linear@10": 0.8261591002253297, - "ndcg_exponential@1": 0.673, - "ndcg_exponential@5": 0.8099367287839547, - "ndcg_exponential@10": 0.8261591002253297, - "precision@1": 0.673, - "precision@5": 0.18280000000000002, - "precision@10": 0.09640000000000001, - "recall@1": 0.673, - "recall@5": 0.914, - "recall@10": 0.964, - "f1@1": 0.673, - "f1@5": 0.3046666666666667, - "f1@10": 0.17527272727272725, - "map@1": 0.673, - "map@5": 0.7745833333333334, - "map@10": 0.7813182539682539, - "mrr@1": 0.673, - "mrr@5": 0.7745833333333334, - "mrr@10": 0.7813182539682539, + "hit@1": 0.67, + "hit@5": 0.915, + "hit@10": 0.967, + "ndcg_linear@1": 0.67, + "ndcg_linear@5": 0.8091032236591382, + "ndcg_linear@10": 0.8259001985564557, + "ndcg_exponential@1": 0.67, + "ndcg_exponential@5": 0.8091032236591382, + "ndcg_exponential@10": 0.8259001985564557, + "precision@1": 0.67, + "precision@5": 0.18300000000000002, + "precision@10": 0.09670000000000002, + "recall@1": 0.67, + "recall@5": 0.915, + "recall@10": 0.967, + "f1@1": 0.67, + "f1@5": 0.305, + "f1@10": 0.17581818181818182, + "map@1": 0.67, + "map@5": 0.7731499999999999, + "map@10": 0.7800837301587302, + "mrr@1": 0.67, + "mrr@5": 0.7731499999999999, + "mrr@10": 0.7800837301587302, "num_pred": 1000, "num_data": 1000 }, "Place365": { - "hit@1": 0.358, - "hit@5": 0.668, - "hit@10": 0.773, - "ndcg_linear@1": 0.358, - "ndcg_linear@5": 0.522620767615663, - "ndcg_linear@10": 0.5566245695213274, - "ndcg_exponential@1": 0.358, - "ndcg_exponential@5": 0.522620767615663, - "ndcg_exponential@10": 0.5566245695213274, - "precision@1": 0.358, - "precision@5": 0.1336, - "precision@10": 0.07730000000000001, - "recall@1": 0.358, - "recall@5": 0.668, - "recall@10": 0.773, - "f1@1": 0.358, - "f1@5": 0.22266666666666673, - "f1@10": 0.14054545454545456, - "map@1": 0.358, - "map@5": 0.4742833333333333, - "map@10": 0.48834523809523805, - "mrr@1": 0.358, - "mrr@5": 0.4742833333333333, - "mrr@10": 0.48834523809523805, + "hit@1": 0.361, + "hit@5": 0.667, + "hit@10": 0.772, + "ndcg_linear@1": 0.361, + "ndcg_linear@5": 0.5242303273303291, + "ndcg_linear@10": 0.5581312766096997, + "ndcg_exponential@1": 0.361, + "ndcg_exponential@5": 0.5242303273303291, + "ndcg_exponential@10": 0.5581312766096997, + "precision@1": 0.361, + "precision@5": 0.1334, + "precision@10": 0.07719999999999999, + "recall@1": 0.361, + "recall@5": 0.667, + "recall@10": 0.772, + "f1@1": 0.361, + "f1@5": 0.2223333333333334, + "f1@10": 0.14036363636363638, + "map@1": 0.361, + "map@5": 0.4766666666666667, + "map@10": 0.4906230158730158, + "mrr@1": 0.361, + "mrr@5": 0.4766666666666667, + "mrr@10": 0.4906230158730158, "num_pred": 1000, "num_data": 1000 }, "ImageNet-A": { - "hit@1": 0.288, - "hit@5": 0.559, - "hit@10": 0.658, - "ndcg_linear@1": 0.288, - "ndcg_linear@5": 0.432876508364864, - "ndcg_linear@10": 0.46516116229889143, - "ndcg_exponential@1": 0.288, - "ndcg_exponential@5": 0.432876508364864, - "ndcg_exponential@10": 0.46516116229889143, - "precision@1": 0.288, - "precision@5": 0.1118, - "precision@10": 0.0658, - "recall@1": 0.288, - "recall@5": 0.559, - "recall@10": 0.658, - "f1@1": 0.288, - "f1@5": 0.18633333333333338, - "f1@10": 0.11963636363636365, - "map@1": 0.288, - "map@5": 0.3908833333333333, - "map@10": 0.40436825396825393, - "mrr@1": 0.288, - "mrr@5": 0.3908833333333333, - "mrr@10": 0.40436825396825393, + "hit@1": 0.284, + "hit@5": 0.563, + "hit@10": 0.657, + "ndcg_linear@1": 0.284, + "ndcg_linear@5": 0.433263387296594, + "ndcg_linear@10": 0.46385708748332194, + "ndcg_exponential@1": 0.284, + "ndcg_exponential@5": 0.433263387296594, + "ndcg_exponential@10": 0.46385708748332194, + "precision@1": 0.284, + "precision@5": 0.11259999999999999, + "precision@10": 0.0657, + "recall@1": 0.284, + "recall@5": 0.563, + "recall@10": 0.657, + "f1@1": 0.284, + "f1@5": 0.1876666666666667, + "f1@10": 0.11945454545454547, + "map@1": 0.284, + "map@5": 0.3901, + "map@10": 0.4028424603174603, + "mrr@1": 0.284, + "mrr@5": 0.3901, + "mrr@10": 0.4028424603174603, "num_pred": 1000, "num_data": 1000 }, "ImageNet-R": { - "hit@1": 0.786, - "hit@5": 0.95, - "hit@10": 0.969, - "ndcg_linear@1": 0.786, - "ndcg_linear@5": 0.8734468881131793, - "ndcg_linear@10": 0.8795864415830221, - "ndcg_exponential@1": 0.786, - "ndcg_exponential@5": 0.8734468881131793, - "ndcg_exponential@10": 0.8795864415830221, - "precision@1": 0.786, - "precision@5": 0.19000000000000003, - "precision@10": 0.0969, - "recall@1": 0.786, - "recall@5": 0.95, - "recall@10": 0.969, - "f1@1": 0.786, - "f1@5": 0.31666666666666665, - "f1@10": 0.1761818181818182, - "map@1": 0.786, - "map@5": 0.8479, - "map@10": 0.8504329365079365, - "mrr@1": 0.786, - "mrr@5": 0.8479, - "mrr@10": 0.8504329365079365, + "hit@1": 0.785, + "hit@5": 0.951, + "hit@10": 0.967, + "ndcg_linear@1": 0.785, + "ndcg_linear@5": 0.8730896639740768, + "ndcg_linear@10": 0.8781664991697973, + "ndcg_exponential@1": 0.785, + "ndcg_exponential@5": 0.8730896639740768, + "ndcg_exponential@10": 0.8781664991697973, + "precision@1": 0.785, + "precision@5": 0.1902, + "precision@10": 0.09670000000000002, + "recall@1": 0.785, + "recall@5": 0.951, + "recall@10": 0.967, + "f1@1": 0.785, + "f1@5": 0.317, + "f1@10": 0.17581818181818182, + "map@1": 0.785, + "map@5": 0.8471333333333334, + "map@10": 0.8491730158730159, + "mrr@1": 0.785, + "mrr@5": 0.8471333333333334, + "mrr@10": 0.8491730158730159, "num_pred": 1000, "num_data": 1000 }, "ObjectNet": { - "hit@1": 0.706, - "hit@5": 0.879, + "hit@1": 0.712, + "hit@5": 0.881, "hit@10": 0.924, - "ndcg_linear@1": 0.706, - "ndcg_linear@5": 0.8038263132931281, - "ndcg_linear@10": 0.8180970192061363, - "ndcg_exponential@1": 0.706, - "ndcg_exponential@5": 0.8038263132931281, - "ndcg_exponential@10": 0.8180970192061363, - "precision@1": 0.706, - "precision@5": 0.1758, + "ndcg_linear@1": 0.712, + "ndcg_linear@5": 0.8064388921873149, + "ndcg_linear@10": 0.8200955729396492, + "ndcg_exponential@1": 0.712, + "ndcg_exponential@5": 0.8064388921873149, + "ndcg_exponential@10": 0.8200955729396492, + "precision@1": 0.712, + "precision@5": 0.1762, "precision@10": 0.09240000000000001, - "recall@1": 0.706, - "recall@5": 0.879, + "recall@1": 0.712, + "recall@5": 0.881, "recall@10": 0.924, - "f1@1": 0.706, - "f1@5": 0.293, + "f1@1": 0.712, + "f1@5": 0.2936666666666667, "f1@10": 0.168, - "map@1": 0.706, - "map@5": 0.7783333333333332, - "map@10": 0.7840650793650793, - "mrr@1": 0.706, - "mrr@5": 0.7783333333333332, - "mrr@10": 0.7840650793650793, + "map@1": 0.712, + "map@5": 0.7812833333333333, + "map@10": 0.786781746031746, + "mrr@1": 0.712, + "mrr@5": 0.7812833333333333, + "mrr@10": 0.786781746031746, "num_pred": 1000, "num_data": 1000 }, "Country211": { - "hit@1": 0.265, - "hit@5": 0.473, - "hit@10": 0.574, - "ndcg_linear@1": 0.265, - "ndcg_linear@5": 0.3731346216215207, - "ndcg_linear@10": 0.4058179799007638, - "ndcg_exponential@1": 0.265, - "ndcg_exponential@5": 0.3731346216215207, - "ndcg_exponential@10": 0.4058179799007638, - "precision@1": 0.265, - "precision@5": 0.09459999999999999, - "precision@10": 0.05739999999999999, - "recall@1": 0.265, - "recall@5": 0.473, - "recall@10": 0.574, - "f1@1": 0.265, - "f1@5": 0.1576666666666667, - "f1@10": 0.10436363636363638, - "map@1": 0.265, - "map@5": 0.34014999999999995, - "map@10": 0.3536492063492063, - "mrr@1": 0.265, - "mrr@5": 0.34014999999999995, - "mrr@10": 0.3536492063492063, + "hit@1": 0.264, + "hit@5": 0.474, + "hit@10": 0.577, + "ndcg_linear@1": 0.264, + "ndcg_linear@5": 0.373175555573172, + "ndcg_linear@10": 0.4065111858004858, + "ndcg_exponential@1": 0.264, + "ndcg_exponential@5": 0.373175555573172, + "ndcg_exponential@10": 0.4065111858004858, + "precision@1": 0.264, + "precision@5": 0.09479999999999998, + "precision@10": 0.0577, + "recall@1": 0.264, + "recall@5": 0.474, + "recall@10": 0.577, + "f1@1": 0.264, + "f1@5": 0.15800000000000006, + "f1@10": 0.10490909090909094, + "map@1": 0.264, + "map@5": 0.33985000000000004, + "map@10": 0.35363015873015874, + "mrr@1": 0.264, + "mrr@5": 0.33985000000000004, + "mrr@10": 0.35363015873015874, "num_pred": 1000, "num_data": 1000 }, "OK-VQA": { - "hit@1": 0.299, - "hit@5": 0.591, - "hit@10": 0.698, - "ndcg_linear@1": 0.299, - "ndcg_linear@5": 0.45303291100988974, - "ndcg_linear@10": 0.48733709485734505, - "ndcg_exponential@1": 0.299, - "ndcg_exponential@5": 0.45303291100988974, - "ndcg_exponential@10": 0.48733709485734505, - "precision@1": 0.299, - "precision@5": 0.11819999999999999, - "precision@10": 0.0698, - "recall@1": 0.299, - "recall@5": 0.591, - "recall@10": 0.698, - "f1@1": 0.299, - "f1@5": 0.19700000000000006, - "f1@10": 0.12690909090909094, - "map@1": 0.299, - "map@5": 0.4072166666666666, - "map@10": 0.42119880952380945, - "mrr@1": 0.299, - "mrr@5": 0.4072166666666666, - "mrr@10": 0.42119880952380945, + "hit@1": 0.301, + "hit@5": 0.587, + "hit@10": 0.696, + "ndcg_linear@1": 0.301, + "ndcg_linear@5": 0.4531146488577211, + "ndcg_linear@10": 0.4882241557213494, + "ndcg_exponential@1": 0.301, + "ndcg_exponential@5": 0.4531146488577211, + "ndcg_exponential@10": 0.4882241557213494, + "precision@1": 0.301, + "precision@5": 0.11739999999999999, + "precision@10": 0.0696, + "recall@1": 0.301, + "recall@5": 0.587, + "recall@10": 0.696, + "f1@1": 0.301, + "f1@5": 0.1956666666666667, + "f1@10": 0.12654545454545457, + "map@1": 0.301, + "map@5": 0.40854999999999997, + "map@10": 0.4229650793650793, + "mrr@1": 0.301, + "mrr@5": 0.40854999999999997, + "mrr@10": 0.4229650793650793, "num_pred": 1000, "num_data": 1000 }, "A-OKVQA": { "hit@1": 0.186, - "hit@5": 0.465, - "hit@10": 0.598, + "hit@5": 0.462, + "hit@10": 0.599, "ndcg_linear@1": 0.186, - "ndcg_linear@5": 0.3302483880115797, - "ndcg_linear@10": 0.3728925871674892, + "ndcg_linear@5": 0.32919091135211725, + "ndcg_linear@10": 0.3732072317767905, "ndcg_exponential@1": 0.186, - "ndcg_exponential@5": 0.3302483880115797, - "ndcg_exponential@10": 0.3728925871674892, + "ndcg_exponential@5": 0.32919091135211725, + "ndcg_exponential@10": 0.3732072317767905, "precision@1": 0.186, - "precision@5": 0.09299999999999999, - "precision@10": 0.05979999999999999, + "precision@5": 0.0924, + "precision@10": 0.059899999999999995, "recall@1": 0.186, - "recall@5": 0.465, - "recall@10": 0.598, + "recall@5": 0.462, + "recall@10": 0.599, "f1@1": 0.186, - "f1@5": 0.15500000000000005, - "f1@10": 0.10872727272727274, + "f1@5": 0.15400000000000005, + "f1@10": 0.10890909090909093, "map@1": 0.186, - "map@5": 0.2858, - "map@10": 0.30318611111111116, + "map@5": 0.28529999999999994, + "map@10": 0.3033, "mrr@1": 0.186, - "mrr@5": 0.2858, - "mrr@10": 0.30318611111111116, + "mrr@5": 0.28529999999999994, + "mrr@10": 0.3033, "num_pred": 1000, "num_data": 1000 }, "DocVQA": { - "hit@1": 0.298, - "hit@5": 0.482, + "hit@1": 0.3, + "hit@5": 0.487, "hit@10": 0.571, - "ndcg_linear@1": 0.298, - "ndcg_linear@5": 0.3957691402034676, - "ndcg_linear@10": 0.4242933720175296, - "ndcg_exponential@1": 0.298, - "ndcg_exponential@5": 0.3957691402034676, - "ndcg_exponential@10": 0.4242933720175296, - "precision@1": 0.298, - "precision@5": 0.0964, + "ndcg_linear@1": 0.3, + "ndcg_linear@5": 0.3986168397358528, + "ndcg_linear@10": 0.425625078860842, + "ndcg_exponential@1": 0.3, + "ndcg_exponential@5": 0.3986168397358528, + "ndcg_exponential@10": 0.425625078860842, + "precision@1": 0.3, + "precision@5": 0.09739999999999999, "precision@10": 0.05709999999999999, - "recall@1": 0.298, - "recall@5": 0.482, + "recall@1": 0.3, + "recall@5": 0.487, "recall@10": 0.571, - "f1@1": 0.298, - "f1@5": 0.16066666666666668, + "f1@1": 0.3, + "f1@5": 0.16233333333333338, "f1@10": 0.10381818181818184, - "map@1": 0.298, - "map@5": 0.36710000000000004, - "map@10": 0.3787162698412699, - "mrr@1": 0.298, - "mrr@5": 0.36710000000000004, - "mrr@10": 0.3787162698412699, + "map@1": 0.3, + "map@5": 0.36929999999999996, + "map@10": 0.3803484126984127, + "mrr@1": 0.3, + "mrr@5": 0.36929999999999996, + "mrr@10": 0.3803484126984127, "num_pred": 1000, "num_data": 1000 }, "InfographicsVQA": { - "hit@1": 0.116, - "hit@5": 0.276, - "hit@10": 0.355, - "ndcg_linear@1": 0.116, - "ndcg_linear@5": 0.20041421211561675, - "ndcg_linear@10": 0.22615443861016019, - "ndcg_exponential@1": 0.116, - "ndcg_exponential@5": 0.20041421211561675, - "ndcg_exponential@10": 0.22615443861016019, - "precision@1": 0.116, - "precision@5": 0.055200000000000006, - "precision@10": 0.0355, - "recall@1": 0.116, - "recall@5": 0.276, - "recall@10": 0.355, - "f1@1": 0.116, - "f1@5": 0.09200000000000001, - "f1@10": 0.06454545454545456, - "map@1": 0.116, - "map@5": 0.1753333333333333, - "map@10": 0.18606507936507935, - "mrr@1": 0.116, - "mrr@5": 0.1753333333333333, - "mrr@10": 0.18606507936507935, + "hit@1": 0.118, + "hit@5": 0.278, + "hit@10": 0.356, + "ndcg_linear@1": 0.118, + "ndcg_linear@5": 0.20168198127660603, + "ndcg_linear@10": 0.22697387139964162, + "ndcg_exponential@1": 0.118, + "ndcg_exponential@5": 0.20168198127660603, + "ndcg_exponential@10": 0.22697387139964162, + "precision@1": 0.118, + "precision@5": 0.055600000000000004, + "precision@10": 0.0356, + "recall@1": 0.118, + "recall@5": 0.278, + "recall@10": 0.356, + "f1@1": 0.118, + "f1@5": 0.09266666666666669, + "f1@10": 0.06472727272727273, + "map@1": 0.118, + "map@5": 0.17643333333333333, + "map@10": 0.18691111111111108, + "mrr@1": 0.118, + "mrr@5": 0.17643333333333333, + "mrr@10": 0.18691111111111108, "num_pred": 1000, "num_data": 1000 }, "ChartQA": { - "hit@1": 0.134, - "hit@5": 0.218, - "hit@10": 0.245, - "ndcg_linear@1": 0.134, - "ndcg_linear@5": 0.1788395722007737, - "ndcg_linear@10": 0.18758362872612877, - "ndcg_exponential@1": 0.134, - "ndcg_exponential@5": 0.1788395722007737, - "ndcg_exponential@10": 0.18758362872612877, - "precision@1": 0.134, - "precision@5": 0.0436, - "precision@10": 0.0245, - "recall@1": 0.134, - "recall@5": 0.218, - "recall@10": 0.245, - "f1@1": 0.134, - "f1@5": 0.07266666666666668, - "f1@10": 0.04454545454545455, - "map@1": 0.134, - "map@5": 0.16581666666666667, - "map@10": 0.1694349206349206, - "mrr@1": 0.134, - "mrr@5": 0.16581666666666667, - "mrr@10": 0.1694349206349206, + "hit@1": 0.133, + "hit@5": 0.217, + "hit@10": 0.241, + "ndcg_linear@1": 0.133, + "ndcg_linear@5": 0.17758542982391254, + "ndcg_linear@10": 0.18544089721707316, + "ndcg_exponential@1": 0.133, + "ndcg_exponential@5": 0.17758542982391254, + "ndcg_exponential@10": 0.18544089721707316, + "precision@1": 0.133, + "precision@5": 0.04340000000000001, + "precision@10": 0.0241, + "recall@1": 0.133, + "recall@5": 0.217, + "recall@10": 0.241, + "f1@1": 0.133, + "f1@5": 0.07233333333333335, + "f1@10": 0.04381818181818182, + "map@1": 0.133, + "map@5": 0.16448333333333334, + "map@10": 0.16778253968253967, + "mrr@1": 0.133, + "mrr@5": 0.16448333333333334, + "mrr@10": 0.16778253968253967, "num_pred": 1000, "num_data": 1000 }, "Visual7W": { - "hit@1": 0.162, - "hit@5": 0.468, + "hit@1": 0.156, + "hit@5": 0.472, "hit@10": 0.597, - "ndcg_linear@1": 0.162, - "ndcg_linear@5": 0.32148658142856573, - "ndcg_linear@10": 0.36302388780471345, - "ndcg_exponential@1": 0.162, - "ndcg_exponential@5": 0.32148658142856573, - "ndcg_exponential@10": 0.36302388780471345, - "precision@1": 0.162, - "precision@5": 0.09359999999999999, - "precision@10": 0.059699999999999996, - "recall@1": 0.162, - "recall@5": 0.468, + "ndcg_linear@1": 0.156, + "ndcg_linear@5": 0.320305917595171, + "ndcg_linear@10": 0.3605247347197894, + "ndcg_exponential@1": 0.156, + "ndcg_exponential@5": 0.320305917595171, + "ndcg_exponential@10": 0.3605247347197894, + "precision@1": 0.156, + "precision@5": 0.0944, + "precision@10": 0.05969999999999999, + "recall@1": 0.156, + "recall@5": 0.472, "recall@10": 0.597, - "f1@1": 0.162, - "f1@5": 0.15600000000000006, + "f1@1": 0.156, + "f1@5": 0.15733333333333338, "f1@10": 0.10854545454545456, - "map@1": 0.162, - "map@5": 0.27301666666666663, - "map@10": 0.2900488095238095, - "mrr@1": 0.162, - "mrr@5": 0.27301666666666663, - "mrr@10": 0.2900488095238095, + "map@1": 0.156, + "map@5": 0.27018333333333333, + "map@10": 0.2866519841269841, + "mrr@1": 0.156, + "mrr@5": 0.27018333333333333, + "mrr@10": 0.2866519841269841, "num_pred": 1000, "num_data": 1000 }, "ScienceQA": { - "hit@1": 0.273, - "hit@5": 0.584, - "hit@10": 0.69, - "ndcg_linear@1": 0.273, - "ndcg_linear@5": 0.4400970760761488, - "ndcg_linear@10": 0.47441367522579586, - "ndcg_exponential@1": 0.273, - "ndcg_exponential@5": 0.4400970760761488, - "ndcg_exponential@10": 0.47441367522579586, - "precision@1": 0.273, - "precision@5": 0.1168, - "precision@10": 0.069, - "recall@1": 0.273, - "recall@5": 0.584, - "recall@10": 0.69, - "f1@1": 0.273, - "f1@5": 0.19466666666666674, - "f1@10": 0.12545454545454546, - "map@1": 0.273, - "map@5": 0.392, - "map@10": 0.40618293650793647, - "mrr@1": 0.273, - "mrr@5": 0.392, - "mrr@10": 0.40618293650793647, + "hit@1": 0.271, + "hit@5": 0.586, + "hit@10": 0.696, + "ndcg_linear@1": 0.271, + "ndcg_linear@5": 0.43966226986589296, + "ndcg_linear@10": 0.47523284421299805, + "ndcg_exponential@1": 0.271, + "ndcg_exponential@5": 0.43966226986589296, + "ndcg_exponential@10": 0.47523284421299805, + "precision@1": 0.271, + "precision@5": 0.11719999999999998, + "precision@10": 0.0696, + "recall@1": 0.271, + "recall@5": 0.586, + "recall@10": 0.696, + "f1@1": 0.271, + "f1@5": 0.19533333333333336, + "f1@10": 0.12654545454545457, + "map@1": 0.271, + "map@5": 0.39083333333333337, + "map@10": 0.4055134920634921, + "mrr@1": 0.271, + "mrr@5": 0.39083333333333337, + "mrr@10": 0.4055134920634921, "num_pred": 1000, "num_data": 1000 }, "VizWiz": { - "hit@1": 0.37, - "hit@5": 0.591, - "hit@10": 0.659, - "ndcg_linear@1": 0.37, - "ndcg_linear@5": 0.48838071582352677, - "ndcg_linear@10": 0.5107500110876547, - "ndcg_exponential@1": 0.37, - "ndcg_exponential@5": 0.48838071582352677, - "ndcg_exponential@10": 0.5107500110876547, - "precision@1": 0.37, - "precision@5": 0.11819999999999999, - "precision@10": 0.0659, - "recall@1": 0.37, - "recall@5": 0.591, - "recall@10": 0.659, - "f1@1": 0.37, - "f1@5": 0.19700000000000006, - "f1@10": 0.11981818181818184, - "map@1": 0.37, - "map@5": 0.4541666666666666, - "map@10": 0.4636186507936508, - "mrr@1": 0.37, - "mrr@5": 0.4541666666666666, - "mrr@10": 0.4636186507936508, + "hit@1": 0.371, + "hit@5": 0.59, + "hit@10": 0.66, + "ndcg_linear@1": 0.371, + "ndcg_linear@5": 0.48902584065980487, + "ndcg_linear@10": 0.5120006294906867, + "ndcg_exponential@1": 0.371, + "ndcg_exponential@5": 0.48902584065980487, + "ndcg_exponential@10": 0.5120006294906867, + "precision@1": 0.371, + "precision@5": 0.11799999999999998, + "precision@10": 0.066, + "recall@1": 0.371, + "recall@5": 0.59, + "recall@10": 0.66, + "f1@1": 0.371, + "f1@5": 0.1966666666666667, + "f1@10": 0.12000000000000001, + "map@1": 0.371, + "map@5": 0.4552833333333333, + "map@10": 0.4649615079365079, + "mrr@1": 0.371, + "mrr@5": 0.4552833333333333, + "mrr@10": 0.4649615079365079, "num_pred": 1000, "num_data": 1000 }, "GQA": { - "hit@1": 0.751, - "hit@5": 0.931, - "hit@10": 0.967, - "ndcg_linear@1": 0.751, - "ndcg_linear@5": 0.8518997657120991, - "ndcg_linear@10": 0.8636150852590038, - "ndcg_exponential@1": 0.751, - "ndcg_exponential@5": 0.8518997657120991, - "ndcg_exponential@10": 0.8636150852590038, - "precision@1": 0.751, - "precision@5": 0.1862, - "precision@10": 0.09670000000000002, - "recall@1": 0.751, - "recall@5": 0.931, - "recall@10": 0.967, - "f1@1": 0.751, - "f1@5": 0.31033333333333335, - "f1@10": 0.17581818181818182, - "map@1": 0.751, - "map@5": 0.8251499999999998, - "map@10": 0.8300325396825396, - "mrr@1": 0.751, - "mrr@5": 0.8251499999999998, - "mrr@10": 0.8300325396825396, + "hit@1": 0.753, + "hit@5": 0.932, + "hit@10": 0.966, + "ndcg_linear@1": 0.753, + "ndcg_linear@5": 0.8527706166353296, + "ndcg_linear@10": 0.8638262292876865, + "ndcg_exponential@1": 0.753, + "ndcg_exponential@5": 0.8527706166353296, + "ndcg_exponential@10": 0.8638262292876865, + "precision@1": 0.753, + "precision@5": 0.1864, + "precision@10": 0.09660000000000002, + "recall@1": 0.753, + "recall@5": 0.932, + "recall@10": 0.966, + "f1@1": 0.753, + "f1@5": 0.3106666666666667, + "f1@10": 0.17563636363636362, + "map@1": 0.753, + "map@5": 0.8260166666666666, + "map@10": 0.8306186507936508, + "mrr@1": 0.753, + "mrr@5": 0.8260166666666666, + "mrr@10": 0.8306186507936508, "num_pred": 1000, "num_data": 1000 }, "TextVQA": { - "hit@1": 0.397, - "hit@5": 0.598, - "hit@10": 0.667, - "ndcg_linear@1": 0.397, - "ndcg_linear@5": 0.5062758279351262, - "ndcg_linear@10": 0.5287712187842828, - "ndcg_exponential@1": 0.397, - "ndcg_exponential@5": 0.5062758279351262, - "ndcg_exponential@10": 0.5287712187842828, - "precision@1": 0.397, - "precision@5": 0.1196, - "precision@10": 0.06670000000000001, - "recall@1": 0.397, - "recall@5": 0.598, - "recall@10": 0.667, - "f1@1": 0.397, - "f1@5": 0.1993333333333334, - "f1@10": 0.12127272727272731, - "map@1": 0.397, - "map@5": 0.4755333333333333, - "map@10": 0.4849186507936508, - "mrr@1": 0.397, - "mrr@5": 0.4755333333333333, - "mrr@10": 0.4849186507936508, + "hit@1": 0.395, + "hit@5": 0.604, + "hit@10": 0.669, + "ndcg_linear@1": 0.395, + "ndcg_linear@5": 0.5083113930567382, + "ndcg_linear@10": 0.5295127796560257, + "ndcg_exponential@1": 0.395, + "ndcg_exponential@5": 0.5083113930567382, + "ndcg_exponential@10": 0.5295127796560257, + "precision@1": 0.395, + "precision@5": 0.12079999999999998, + "precision@10": 0.0669, + "recall@1": 0.395, + "recall@5": 0.604, + "recall@10": 0.669, + "f1@1": 0.395, + "f1@5": 0.20133333333333336, + "f1@10": 0.12163636363636365, + "map@1": 0.395, + "map@5": 0.47626666666666667, + "map@10": 0.48511785714285716, + "mrr@1": 0.395, + "mrr@5": 0.47626666666666667, + "mrr@10": 0.48511785714285716, "num_pred": 1000, "num_data": 1000 }, "VisDial": { - "hit@1": 0.481, - "hit@5": 0.734, - "hit@10": 0.841, - "ndcg_linear@1": 0.481, - "ndcg_linear@5": 0.6145695068804011, - "ndcg_linear@10": 0.6493023243064213, - "ndcg_exponential@1": 0.481, - "ndcg_exponential@5": 0.6145695068804011, - "ndcg_exponential@10": 0.6493023243064213, - "precision@1": 0.481, - "precision@5": 0.1468, - "precision@10": 0.08410000000000001, - "recall@1": 0.481, - "recall@5": 0.734, - "recall@10": 0.841, - "f1@1": 0.481, - "f1@5": 0.24466666666666675, - "f1@10": 0.1529090909090909, - "map@1": 0.481, - "map@5": 0.5748833333333333, - "map@10": 0.5892829365079365, - "mrr@1": 0.481, - "mrr@5": 0.5748833333333333, - "mrr@10": 0.5892829365079365, + "hit@1": 0.477, + "hit@5": 0.736, + "hit@10": 0.843, + "ndcg_linear@1": 0.477, + "ndcg_linear@5": 0.6145547970983828, + "ndcg_linear@10": 0.6491616761542083, + "ndcg_exponential@1": 0.477, + "ndcg_exponential@5": 0.6145547970983828, + "ndcg_exponential@10": 0.6491616761542083, + "precision@1": 0.477, + "precision@5": 0.14720000000000003, + "precision@10": 0.08430000000000001, + "recall@1": 0.477, + "recall@5": 0.736, + "recall@10": 0.843, + "f1@1": 0.477, + "f1@5": 0.24533333333333338, + "f1@10": 0.15327272727272728, + "map@1": 0.477, + "map@5": 0.57415, + "map@10": 0.5884289682539683, + "mrr@1": 0.477, + "mrr@5": 0.57415, + "mrr@10": 0.5884289682539683, "num_pred": 1000, "num_data": 1000 }, "CIRR": { - "hit@1": 0.442, + "hit@1": 0.431, "hit@5": 0.856, - "hit@10": 0.929, - "ndcg_linear@1": 0.442, - "ndcg_linear@5": 0.6719913959349983, - "ndcg_linear@10": 0.695699675833813, - "ndcg_exponential@1": 0.442, - "ndcg_exponential@5": 0.6719913959349983, - "ndcg_exponential@10": 0.695699675833813, - "precision@1": 0.442, - "precision@5": 0.17120000000000002, - "precision@10": 0.09290000000000001, - "recall@1": 0.442, + "hit@10": 0.93, + "ndcg_linear@1": 0.431, + "ndcg_linear@5": 0.6670891349916955, + "ndcg_linear@10": 0.6911930704299414, + "ndcg_exponential@1": 0.431, + "ndcg_exponential@5": 0.6670891349916955, + "ndcg_exponential@10": 0.6911930704299414, + "precision@1": 0.431, + "precision@5": 0.1712, + "precision@10": 0.093, + "recall@1": 0.431, "recall@5": 0.856, - "recall@10": 0.929, - "f1@1": 0.442, + "recall@10": 0.93, + "f1@1": 0.431, "f1@5": 0.2853333333333334, - "f1@10": 0.16890909090909087, - "map@1": 0.442, - "map@5": 0.6098333333333332, - "map@10": 0.619682142857143, - "mrr@1": 0.442, - "mrr@5": 0.6098333333333332, - "mrr@10": 0.619682142857143, + "f1@10": 0.16909090909090907, + "map@1": 0.431, + "map@5": 0.6033166666666666, + "map@10": 0.613363492063492, + "mrr@1": 0.431, + "mrr@5": 0.6033166666666666, + "mrr@10": 0.613363492063492, "num_pred": 1000, "num_data": 1000 }, "VisualNews_t2i": { - "hit@1": 0.747, - "hit@5": 0.885, - "hit@10": 0.918, - "ndcg_linear@1": 0.747, - "ndcg_linear@5": 0.8235708258901523, - "ndcg_linear@10": 0.8342314434199707, - "ndcg_exponential@1": 0.747, - "ndcg_exponential@5": 0.8235708258901523, - "ndcg_exponential@10": 0.8342314434199707, - "precision@1": 0.747, - "precision@5": 0.177, - "precision@10": 0.0918, - "recall@1": 0.747, - "recall@5": 0.885, - "recall@10": 0.918, - "f1@1": 0.747, - "f1@5": 0.295, - "f1@10": 0.1669090909090909, - "map@1": 0.747, - "map@5": 0.80285, - "map@10": 0.8072428571428572, - "mrr@1": 0.747, - "mrr@5": 0.80285, - "mrr@10": 0.8072428571428572, + "hit@1": 0.748, + "hit@5": 0.883, + "hit@10": 0.919, + "ndcg_linear@1": 0.748, + "ndcg_linear@5": 0.8233231614657159, + "ndcg_linear@10": 0.835026093127658, + "ndcg_exponential@1": 0.748, + "ndcg_exponential@5": 0.8233231614657159, + "ndcg_exponential@10": 0.835026093127658, + "precision@1": 0.748, + "precision@5": 0.17660000000000003, + "precision@10": 0.09190000000000001, + "recall@1": 0.748, + "recall@5": 0.883, + "recall@10": 0.919, + "f1@1": 0.748, + "f1@5": 0.2943333333333334, + "f1@10": 0.16709090909090907, + "map@1": 0.748, + "map@5": 0.8031333333333334, + "map@10": 0.8079984126984128, + "mrr@1": 0.748, + "mrr@5": 0.8031333333333334, + "mrr@10": 0.8079984126984128, "num_pred": 1000, "num_data": 1000 }, "VisualNews_i2t": { - "hit@1": 0.783, + "hit@1": 0.777, "hit@5": 0.904, - "hit@10": 0.925, - "ndcg_linear@1": 0.783, - "ndcg_linear@5": 0.8502353309844382, - "ndcg_linear@10": 0.8571131175684055, - "ndcg_exponential@1": 0.783, - "ndcg_exponential@5": 0.8502353309844382, - "ndcg_exponential@10": 0.8571131175684055, - "precision@1": 0.783, + "hit@10": 0.926, + "ndcg_linear@1": 0.777, + "ndcg_linear@5": 0.8476281202451525, + "ndcg_linear@10": 0.8548637518117048, + "ndcg_exponential@1": 0.777, + "ndcg_exponential@5": 0.8476281202451525, + "ndcg_exponential@10": 0.8548637518117048, + "precision@1": 0.777, "precision@5": 0.18080000000000002, - "precision@10": 0.0925, - "recall@1": 0.783, + "precision@10": 0.0926, + "recall@1": 0.777, "recall@5": 0.904, - "recall@10": 0.925, - "f1@1": 0.783, + "recall@10": 0.926, + "f1@1": 0.777, "f1@5": 0.3013333333333334, - "f1@10": 0.16818181818181815, - "map@1": 0.783, - "map@5": 0.8320833333333334, - "map@10": 0.834970634920635, - "mrr@1": 0.783, - "mrr@5": 0.8320833333333334, - "mrr@10": 0.834970634920635, + "f1@10": 0.16836363636363635, + "map@1": 0.777, + "map@5": 0.8285833333333332, + "map@10": 0.8316396825396826, + "mrr@1": 0.777, + "mrr@5": 0.8285833333333332, + "mrr@10": 0.8316396825396826, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_t2i": { - "hit@1": 0.681, - "hit@5": 0.916, + "hit@1": 0.683, + "hit@5": 0.915, "hit@10": 0.956, - "ndcg_linear@1": 0.681, - "ndcg_linear@5": 0.8114477974886068, - "ndcg_linear@10": 0.8247525393057755, - "ndcg_exponential@1": 0.681, - "ndcg_exponential@5": 0.8114477974886068, - "ndcg_exponential@10": 0.8247525393057755, - "precision@1": 0.681, - "precision@5": 0.18320000000000003, + "ndcg_linear@1": 0.683, + "ndcg_linear@5": 0.8117037205422699, + "ndcg_linear@10": 0.8252015417103143, + "ndcg_exponential@1": 0.683, + "ndcg_exponential@5": 0.8117037205422699, + "ndcg_exponential@10": 0.8252015417103143, + "precision@1": 0.683, + "precision@5": 0.183, "precision@10": 0.0956, - "recall@1": 0.681, - "recall@5": 0.916, + "recall@1": 0.683, + "recall@5": 0.915, "recall@10": 0.956, - "f1@1": 0.681, - "f1@5": 0.30533333333333335, + "f1@1": 0.683, + "f1@5": 0.305, "f1@10": 0.17381818181818182, - "map@1": 0.681, - "map@5": 0.7761666666666668, - "map@10": 0.7818726190476191, - "mrr@1": 0.681, - "mrr@5": 0.7761666666666668, - "mrr@10": 0.7818726190476191, + "map@1": 0.683, + "map@5": 0.7768666666666667, + "map@10": 0.7825749999999999, + "mrr@1": 0.683, + "mrr@5": 0.7768666666666667, + "mrr@10": 0.7825749999999999, "num_pred": 1000, "num_data": 1000 }, "MSCOCO_i2t": { - "hit@1": 0.631, - "hit@5": 0.865, - "hit@10": 0.943, - "ndcg_linear@1": 0.631, - "ndcg_linear@5": 0.7574725452263779, - "ndcg_linear@10": 0.7831291095368749, - "ndcg_exponential@1": 0.631, - "ndcg_exponential@5": 0.7574725452263779, - "ndcg_exponential@10": 0.7831291095368749, - "precision@1": 0.631, - "precision@5": 0.173, - "precision@10": 0.09430000000000001, - "recall@1": 0.631, - "recall@5": 0.865, - "recall@10": 0.943, - "f1@1": 0.631, - "f1@5": 0.2883333333333334, - "f1@10": 0.17145454545454544, - "map@1": 0.631, - "map@5": 0.7215499999999999, - "map@10": 0.7323861111111112, - "mrr@1": 0.631, - "mrr@5": 0.7215499999999999, - "mrr@10": 0.7323861111111112, + "hit@1": 0.632, + "hit@5": 0.862, + "hit@10": 0.947, + "ndcg_linear@1": 0.632, + "ndcg_linear@5": 0.7571875350035278, + "ndcg_linear@10": 0.7850689801806203, + "ndcg_exponential@1": 0.632, + "ndcg_exponential@5": 0.7571875350035278, + "ndcg_exponential@10": 0.7850689801806203, + "precision@1": 0.632, + "precision@5": 0.17239999999999997, + "precision@10": 0.09470000000000002, + "recall@1": 0.632, + "recall@5": 0.862, + "recall@10": 0.947, + "f1@1": 0.632, + "f1@5": 0.2873333333333334, + "f1@10": 0.17218181818181816, + "map@1": 0.632, + "map@5": 0.7220666666666666, + "map@10": 0.7338027777777778, + "mrr@1": 0.632, + "mrr@5": 0.7220666666666666, + "mrr@10": 0.7338027777777778, "num_pred": 1000, "num_data": 1000 }, "NIGHTS": { - "hit@1": 0.67, - "hit@5": 0.972, + "hit@1": 0.676, + "hit@5": 0.973, "hit@10": 0.989, - "ndcg_linear@1": 0.67, - "ndcg_linear@5": 0.8448167222624823, - "ndcg_linear@10": 0.8505376817455529, - "ndcg_exponential@1": 0.67, - "ndcg_exponential@5": 0.8448167222624823, - "ndcg_exponential@10": 0.8505376817455529, - "precision@1": 0.67, - "precision@5": 0.19440000000000004, + "ndcg_linear@1": 0.676, + "ndcg_linear@5": 0.8474618202991269, + "ndcg_linear@10": 0.8527977954541133, + "ndcg_exponential@1": 0.676, + "ndcg_exponential@5": 0.8474618202991269, + "ndcg_exponential@10": 0.8527977954541133, + "precision@1": 0.676, + "precision@5": 0.19460000000000002, "precision@10": 0.09890000000000002, - "recall@1": 0.67, - "recall@5": 0.972, + "recall@1": 0.676, + "recall@5": 0.973, "recall@10": 0.989, - "f1@1": 0.67, - "f1@5": 0.324, + "f1@1": 0.676, + "f1@5": 0.3243333333333333, "f1@10": 0.17981818181818182, - "map@1": 0.67, - "map@5": 0.8012, - "map@10": 0.8036956349206349, - "mrr@1": 0.67, - "mrr@5": 0.8012, - "mrr@10": 0.8036956349206349, + "map@1": 0.676, + "map@5": 0.80445, + "map@10": 0.8067484126984127, + "mrr@1": 0.676, + "mrr@5": 0.80445, + "mrr@10": 0.8067484126984127, "num_pred": 1000, "num_data": 1000 }, @@ -769,11 +769,11 @@ "hit@5": 0.983, "hit@10": 0.992, "ndcg_linear@1": 0.888, - "ndcg_linear@5": 0.9435607350102705, - "ndcg_linear@10": 0.9464534316497176, + "ndcg_linear@5": 0.9435684521405523, + "ndcg_linear@10": 0.946484022633774, "ndcg_exponential@1": 0.888, - "ndcg_exponential@5": 0.9435607350102705, - "ndcg_exponential@10": 0.9464534316497176, + "ndcg_exponential@5": 0.9435684521405523, + "ndcg_exponential@10": 0.946484022633774, "precision@1": 0.888, "precision@5": 0.19660000000000002, "precision@10": 0.09920000000000001, @@ -785,1134 +785,1162 @@ "f1@10": 0.18036363636363634, "map@1": 0.888, "map@5": 0.9299833333333334, - "map@10": 0.9311646825396825, + "map@10": 0.9311884920634921, "mrr@1": 0.888, "mrr@5": 0.9299833333333334, - "mrr@10": 0.9311646825396825, + "mrr@10": 0.9311884920634921, "num_pred": 1000, "num_data": 1000 }, "FashionIQ": { - "hit@1": 0.329, - "hit@5": 0.591, - "hit@10": 0.703, - "ndcg_linear@1": 0.329, - "ndcg_linear@5": 0.46700970737005354, - "ndcg_linear@10": 0.5034973715066363, - "ndcg_exponential@1": 0.329, - "ndcg_exponential@5": 0.46700970737005354, - "ndcg_exponential@10": 0.5034973715066363, - "precision@1": 0.329, - "precision@5": 0.11819999999999999, - "precision@10": 0.0703, - "recall@1": 0.329, - "recall@5": 0.591, - "recall@10": 0.703, - "f1@1": 0.329, - "f1@5": 0.19700000000000006, - "f1@10": 0.12781818181818183, - "map@1": 0.329, - "map@5": 0.42585, - "map@10": 0.44107301587301584, - "mrr@1": 0.329, - "mrr@5": 0.42585, - "mrr@10": 0.44107301587301584, + "hit@1": 0.322, + "hit@5": 0.593, + "hit@10": 0.712, + "ndcg_linear@1": 0.322, + "ndcg_linear@5": 0.4652105281889922, + "ndcg_linear@10": 0.5033344685536332, + "ndcg_exponential@1": 0.322, + "ndcg_exponential@5": 0.4652105281889922, + "ndcg_exponential@10": 0.5033344685536332, + "precision@1": 0.322, + "precision@5": 0.1186, + "precision@10": 0.0712, + "recall@1": 0.322, + "recall@5": 0.593, + "recall@10": 0.712, + "f1@1": 0.322, + "f1@5": 0.1976666666666667, + "f1@10": 0.12945454545454546, + "map@1": 0.322, + "map@5": 0.4227666666666666, + "map@10": 0.4382964285714286, + "mrr@1": 0.322, + "mrr@5": 0.4227666666666666, + "mrr@10": 0.4382964285714286, "num_pred": 1000, "num_data": 1000 }, "Wiki-SS-NQ": { - "hit@1": 0.739, - "hit@5": 0.915, + "hit@1": 0.738, + "hit@5": 0.913, "hit@10": 0.936, - "ndcg_linear@1": 0.739, - "ndcg_linear@5": 0.8380472127241243, - "ndcg_linear@10": 0.844767008816438, - "ndcg_exponential@1": 0.739, - "ndcg_exponential@5": 0.8380472127241243, - "ndcg_exponential@10": 0.844767008816438, - "precision@1": 0.739, - "precision@5": 0.183, - "precision@10": 0.0936, - "recall@1": 0.739, - "recall@5": 0.915, + "ndcg_linear@1": 0.738, + "ndcg_linear@5": 0.8370276494865163, + "ndcg_linear@10": 0.8445736479113601, + "ndcg_exponential@1": 0.738, + "ndcg_exponential@5": 0.8370276494865163, + "ndcg_exponential@10": 0.8445736479113601, + "precision@1": 0.738, + "precision@5": 0.1826, + "precision@10": 0.09360000000000002, + "recall@1": 0.738, + "recall@5": 0.913, "recall@10": 0.936, - "f1@1": 0.739, - "f1@5": 0.30500000000000005, - "f1@10": 0.17018181818181818, - "map@1": 0.739, - "map@5": 0.8119500000000001, - "map@10": 0.8146797619047619, - "mrr@1": 0.739, - "mrr@5": 0.8119500000000001, - "mrr@10": 0.8146797619047619, + "f1@1": 0.738, + "f1@5": 0.30433333333333334, + "f1@10": 0.17018181818181816, + "map@1": 0.738, + "map@5": 0.8112166666666667, + "map@10": 0.8143948412698412, + "mrr@1": 0.738, + "mrr@5": 0.8112166666666667, + "mrr@10": 0.8143948412698412, "num_pred": 1000, "num_data": 1000 }, "OVEN": { - "hit@1": 0.723, + "hit@1": 0.721, "hit@5": 0.889, - "hit@10": 0.926, - "ndcg_linear@1": 0.723, - "ndcg_linear@5": 0.8150649423608013, - "ndcg_linear@10": 0.827442822447229, - "ndcg_exponential@1": 0.723, - "ndcg_exponential@5": 0.8150649423608013, - "ndcg_exponential@10": 0.827442822447229, - "precision@1": 0.723, + "hit@10": 0.928, + "ndcg_linear@1": 0.721, + "ndcg_linear@5": 0.8139268369758932, + "ndcg_linear@10": 0.8267391325094817, + "ndcg_exponential@1": 0.721, + "ndcg_exponential@5": 0.8139268369758932, + "ndcg_exponential@10": 0.8267391325094817, + "precision@1": 0.721, "precision@5": 0.1778, - "precision@10": 0.09260000000000003, - "recall@1": 0.723, + "precision@10": 0.09280000000000001, + "recall@1": 0.721, "recall@5": 0.889, - "recall@10": 0.926, - "f1@1": 0.723, + "recall@10": 0.928, + "f1@1": 0.721, "f1@5": 0.2963333333333334, - "f1@10": 0.16836363636363638, - "map@1": 0.723, - "map@5": 0.7901, - "map@10": 0.7954480158730158, - "mrr@1": 0.723, - "mrr@5": 0.7901, - "mrr@10": 0.7954480158730158, + "f1@10": 0.1687272727272727, + "map@1": 0.721, + "map@5": 0.7885833333333334, + "map@10": 0.7939900793650794, + "mrr@1": 0.721, + "mrr@5": 0.7885833333333334, + "mrr@10": 0.7939900793650794, "num_pred": 1000, "num_data": 1000 }, "EDIS": { - "hit@1": 0.918, + "hit@1": 0.917, "hit@5": 0.994, "hit@10": 0.999, - "ndcg_linear@1": 0.918, - "ndcg_linear@5": 0.9628645846673214, - "ndcg_linear@10": 0.9645039533935458, - "ndcg_exponential@1": 0.918, - "ndcg_exponential@5": 0.9628645846673214, - "ndcg_exponential@10": 0.9645039533935458, - "precision@1": 0.918, + "ndcg_linear@1": 0.917, + "ndcg_linear@5": 0.9620589014093397, + "ndcg_linear@10": 0.9637032755327911, + "ndcg_exponential@1": 0.917, + "ndcg_exponential@5": 0.9620589014093397, + "ndcg_exponential@10": 0.9637032755327911, + "precision@1": 0.917, "precision@5": 0.1988, "precision@10": 0.0999, - "recall@1": 0.918, + "recall@1": 0.917, "recall@5": 0.994, "recall@10": 0.999, - "f1@1": 0.918, + "f1@1": 0.917, "f1@5": 0.33133333333333337, "f1@10": 0.18163636363636362, - "map@1": 0.918, - "map@5": 0.9521166666666666, - "map@10": 0.9528051587301586, - "mrr@1": 0.918, - "mrr@5": 0.9521166666666666, - "mrr@10": 0.9528051587301586, + "map@1": 0.917, + "map@5": 0.9510666666666667, + "map@10": 0.9517611111111112, + "mrr@1": 0.917, + "mrr@5": 0.9510666666666667, + "mrr@10": 0.9517611111111112, "num_pred": 1000, "num_data": 1000 }, "MSCOCO": { - "hit@1": 0.286, - "hit@5": 0.465, - "hit@10": 0.565, - "ndcg_linear@1": 0.286, - "ndcg_linear@5": 0.3813740287670062, - "ndcg_linear@10": 0.4135276980014484, - "ndcg_exponential@1": 0.286, - "ndcg_exponential@5": 0.3813740287670062, - "ndcg_exponential@10": 0.4135276980014484, - "precision@1": 0.286, - "precision@5": 0.09299999999999999, - "precision@10": 0.056499999999999995, - "recall@1": 0.286, - "recall@5": 0.465, - "recall@10": 0.565, - "f1@1": 0.286, - "f1@5": 0.15500000000000003, - "f1@10": 0.10272727272727275, - "map@1": 0.286, - "map@5": 0.3535833333333333, - "map@10": 0.36674603174603176, - "mrr@1": 0.286, - "mrr@5": 0.3535833333333333, - "mrr@10": 0.36674603174603176, + "hit@1": 0.284, + "hit@5": 0.469, + "hit@10": 0.569, + "ndcg_linear@1": 0.284, + "ndcg_linear@5": 0.38124075003805424, + "ndcg_linear@10": 0.4133179400490789, + "ndcg_exponential@1": 0.284, + "ndcg_exponential@5": 0.38124075003805424, + "ndcg_exponential@10": 0.4133179400490789, + "precision@1": 0.284, + "precision@5": 0.0938, + "precision@10": 0.05689999999999999, + "recall@1": 0.284, + "recall@5": 0.469, + "recall@10": 0.569, + "f1@1": 0.284, + "f1@5": 0.15633333333333338, + "f1@10": 0.10345454545454547, + "map@1": 0.284, + "map@5": 0.35220000000000007, + "map@10": 0.3652853174603175, + "mrr@1": 0.284, + "mrr@5": 0.35220000000000007, + "mrr@10": 0.3652853174603175, "num_pred": 1000, "num_data": 1000 }, "RefCOCO": { - "hit@1": 0.559, - "hit@5": 0.812, - "hit@10": 0.888, - "ndcg_linear@1": 0.559, - "ndcg_linear@5": 0.696070982637125, - "ndcg_linear@10": 0.7209435661259483, - "ndcg_exponential@1": 0.559, - "ndcg_exponential@5": 0.696070982637125, - "ndcg_exponential@10": 0.7209435661259483, - "precision@1": 0.559, - "precision@5": 0.16240000000000002, - "precision@10": 0.08880000000000002, - "recall@1": 0.559, - "recall@5": 0.812, - "recall@10": 0.888, - "f1@1": 0.559, - "f1@5": 0.2706666666666667, - "f1@10": 0.16145454545454543, - "map@1": 0.559, - "map@5": 0.6572833333333333, - "map@10": 0.6677174603174604, - "mrr@1": 0.559, - "mrr@5": 0.6572833333333333, - "mrr@10": 0.6677174603174604, + "hit@1": 0.558, + "hit@5": 0.815, + "hit@10": 0.887, + "ndcg_linear@1": 0.558, + "ndcg_linear@5": 0.6967504066175247, + "ndcg_linear@10": 0.7202607210370956, + "ndcg_exponential@1": 0.558, + "ndcg_exponential@5": 0.6967504066175247, + "ndcg_exponential@10": 0.7202607210370956, + "precision@1": 0.558, + "precision@5": 0.163, + "precision@10": 0.0887, + "recall@1": 0.558, + "recall@5": 0.815, + "recall@10": 0.887, + "f1@1": 0.558, + "f1@5": 0.2716666666666667, + "f1@10": 0.16127272727272726, + "map@1": 0.558, + "map@5": 0.6572166666666667, + "map@10": 0.6670511904761905, + "mrr@1": 0.558, + "mrr@5": 0.6572166666666667, + "mrr@10": 0.6670511904761905, "num_pred": 1000, "num_data": 1000 }, "RefCOCO-Matching": { - "hit@1": 0.733, - "hit@5": 0.989, - "hit@10": 0.998, - "ndcg_linear@1": 0.733, - "ndcg_linear@5": 0.8942489549761802, - "ndcg_linear@10": 0.8972579752593925, - "ndcg_exponential@1": 0.733, - "ndcg_exponential@5": 0.8942489549761802, - "ndcg_exponential@10": 0.8972579752593925, - "precision@1": 0.733, - "precision@5": 0.1978, - "precision@10": 0.09980000000000001, - "recall@1": 0.733, - "recall@5": 0.989, - "recall@10": 0.998, - "f1@1": 0.733, - "f1@5": 0.3296666666666666, - "f1@10": 0.18145454545454545, - "map@1": 0.733, - "map@5": 0.8612833333333333, - "map@10": 0.8625829365079365, - "mrr@1": 0.733, - "mrr@5": 0.8582833333333333, - "mrr@10": 0.8595829365079365, + "hit@1": 0.737, + "hit@5": 0.988, + "hit@10": 0.999, + "ndcg_linear@1": 0.737, + "ndcg_linear@5": 0.8949195527039124, + "ndcg_linear@10": 0.8985594101194291, + "ndcg_exponential@1": 0.737, + "ndcg_exponential@5": 0.8949195527039124, + "ndcg_exponential@10": 0.8985594101194291, + "precision@1": 0.737, + "precision@5": 0.19760000000000003, + "precision@10": 0.0999, + "recall@1": 0.737, + "recall@5": 0.988, + "recall@10": 0.999, + "f1@1": 0.737, + "f1@5": 0.3293333333333333, + "f1@10": 0.18163636363636362, + "map@1": 0.737, + "map@5": 0.8625666666666667, + "map@10": 0.8641190476190476, + "mrr@1": 0.737, + "mrr@5": 0.8595666666666667, + "mrr@10": 0.8611190476190476, "num_pred": 1000, "num_data": 1000 }, "Visual7W-Pointing": { - "hit@1": 0.641, - "hit@5": 0.845, + "hit@1": 0.646, + "hit@5": 0.849, "hit@10": 0.903, - "ndcg_linear@1": 0.641, - "ndcg_linear@5": 0.7524825590830492, - "ndcg_linear@10": 0.7713285704115274, - "ndcg_exponential@1": 0.641, - "ndcg_exponential@5": 0.7524825590830492, - "ndcg_exponential@10": 0.7713285704115274, - "precision@1": 0.641, - "precision@5": 0.169, + "ndcg_linear@1": 0.646, + "ndcg_linear@5": 0.7558859284735994, + "ndcg_linear@10": 0.7732254411904662, + "ndcg_exponential@1": 0.646, + "ndcg_exponential@5": 0.7558859284735994, + "ndcg_exponential@10": 0.7732254411904662, + "precision@1": 0.646, + "precision@5": 0.1698, "precision@10": 0.0903, - "recall@1": 0.641, - "recall@5": 0.845, + "recall@1": 0.646, + "recall@5": 0.849, "recall@10": 0.903, - "f1@1": 0.641, - "f1@5": 0.2816666666666667, + "f1@1": 0.646, + "f1@5": 0.283, "f1@10": 0.16418181818181818, - "map@1": 0.641, - "map@5": 0.7213999999999999, - "map@10": 0.7292277777777778, - "mrr@1": 0.641, - "mrr@5": 0.7213999999999999, - "mrr@10": 0.7292277777777778, + "map@1": 0.646, + "map@5": 0.7247166666666666, + "map@10": 0.7318, + "mrr@1": 0.646, + "mrr@5": 0.7247166666666666, + "mrr@10": 0.7318, "num_pred": 1000, "num_data": 1000 } }, "video": { "K700": { - "hit@1": 0.352, + "hit@1": 0.349, "hit@5": 0.558, - "hit@10": 0.634, - "ndcg_linear@1": 0.352, - "ndcg_linear@5": 0.4624292100503551, - "ndcg_linear@10": 0.4871702819761314, - "ndcg_exponential@1": 0.352, - "ndcg_exponential@5": 0.4624292100503551, - "ndcg_exponential@10": 0.4871702819761314, - "precision@1": 0.352, + "hit@10": 0.636, + "ndcg_linear@1": 0.349, + "ndcg_linear@5": 0.4611993281867249, + "ndcg_linear@10": 0.4864365564552001, + "ndcg_exponential@1": 0.349, + "ndcg_exponential@5": 0.4611993281867249, + "ndcg_exponential@10": 0.4864365564552001, + "precision@1": 0.349, "precision@5": 0.11159999999999999, - "precision@10": 0.06339999999999998, - "recall@1": 0.352, + "precision@10": 0.06359999999999999, + "recall@1": 0.349, "recall@5": 0.558, - "recall@10": 0.634, - "f1@1": 0.352, + "recall@10": 0.636, + "f1@1": 0.349, "f1@5": 0.18600000000000005, - "f1@10": 0.11527272727272728, - "map@1": 0.352, - "map@5": 0.4306, - "map@10": 0.440915873015873, - "mrr@1": 0.352, - "mrr@5": 0.4306, - "mrr@10": 0.440915873015873, + "f1@10": 0.11563636363636365, + "map@1": 0.349, + "map@5": 0.4289166666666666, + "map@10": 0.43934246031746027, + "mrr@1": 0.349, + "mrr@5": 0.4289166666666666, + "mrr@10": 0.43934246031746027, "num_pred": 1000, "num_data": 1000 }, "SmthSmthV2": { - "hit@1": 0.299, - "hit@5": 0.528, - "hit@10": 0.622, - "ndcg_linear@1": 0.299, - "ndcg_linear@5": 0.41899491853941084, - "ndcg_linear@10": 0.44961784439388147, - "ndcg_exponential@1": 0.299, - "ndcg_exponential@5": 0.41899491853941084, - "ndcg_exponential@10": 0.44961784439388147, - "precision@1": 0.299, - "precision@5": 0.1056, - "precision@10": 0.0622, - "recall@1": 0.299, - "recall@5": 0.528, - "recall@10": 0.622, - "f1@1": 0.299, - "f1@5": 0.17600000000000005, - "f1@10": 0.11309090909090909, - "map@1": 0.299, - "map@5": 0.38288333333333335, - "map@10": 0.3956539682539683, - "mrr@1": 0.299, - "mrr@5": 0.38288333333333335, - "mrr@10": 0.3956539682539683, + "hit@1": 0.301, + "hit@5": 0.53, + "hit@10": 0.625, + "ndcg_linear@1": 0.301, + "ndcg_linear@5": 0.4206815181511473, + "ndcg_linear@10": 0.4515185088090522, + "ndcg_exponential@1": 0.301, + "ndcg_exponential@5": 0.4206815181511473, + "ndcg_exponential@10": 0.4515185088090522, + "precision@1": 0.301, + "precision@5": 0.10599999999999998, + "precision@10": 0.06249999999999999, + "recall@1": 0.301, + "recall@5": 0.53, + "recall@10": 0.625, + "f1@1": 0.301, + "f1@5": 0.17666666666666672, + "f1@10": 0.11363636363636366, + "map@1": 0.301, + "map@5": 0.3845, + "map@10": 0.39729801587301583, + "mrr@1": 0.301, + "mrr@5": 0.3845, + "mrr@10": 0.39729801587301583, "num_pred": 1000, "num_data": 1000 }, "HMDB51": { - "hit@1": 0.434, - "hit@5": 0.672, - "hit@10": 0.768, - "ndcg_linear@1": 0.434, - "ndcg_linear@5": 0.5564379559041219, - "ndcg_linear@10": 0.587723216959859, - "ndcg_exponential@1": 0.434, - "ndcg_exponential@5": 0.5564379559041219, - "ndcg_exponential@10": 0.587723216959859, - "precision@1": 0.434, - "precision@5": 0.13439999999999996, - "precision@10": 0.0768, - "recall@1": 0.434, - "recall@5": 0.672, - "recall@10": 0.768, - "f1@1": 0.434, - "f1@5": 0.22400000000000006, - "f1@10": 0.13963636363636361, - "map@1": 0.434, - "map@5": 0.5183166666666666, - "map@10": 0.531368253968254, - "mrr@1": 0.434, - "mrr@5": 0.5183166666666666, - "mrr@10": 0.531368253968254, + "hit@1": 0.43, + "hit@5": 0.671, + "hit@10": 0.766, + "ndcg_linear@1": 0.43, + "ndcg_linear@5": 0.5542792042837158, + "ndcg_linear@10": 0.5850955662310028, + "ndcg_exponential@1": 0.43, + "ndcg_exponential@5": 0.5542792042837158, + "ndcg_exponential@10": 0.5850955662310028, + "precision@1": 0.43, + "precision@5": 0.13419999999999999, + "precision@10": 0.07659999999999999, + "recall@1": 0.43, + "recall@5": 0.671, + "recall@10": 0.766, + "f1@1": 0.43, + "f1@5": 0.2236666666666667, + "f1@10": 0.1392727272727273, + "map@1": 0.43, + "map@5": 0.5157666666666667, + "map@10": 0.5285456349206349, + "mrr@1": 0.43, + "mrr@5": 0.5157666666666667, + "mrr@10": 0.5285456349206349, "num_pred": 1000, "num_data": 1000 }, "UCF101": { - "hit@1": 0.524, - "hit@5": 0.745, - "hit@10": 0.821, - "ndcg_linear@1": 0.524, - "ndcg_linear@5": 0.6452214501789469, - "ndcg_linear@10": 0.6701386847278763, - "ndcg_exponential@1": 0.524, - "ndcg_exponential@5": 0.6452214501789469, - "ndcg_exponential@10": 0.6701386847278763, - "precision@1": 0.524, - "precision@5": 0.149, - "precision@10": 0.0821, - "recall@1": 0.524, - "recall@5": 0.745, - "recall@10": 0.821, - "f1@1": 0.524, - "f1@5": 0.2483333333333334, - "f1@10": 0.14927272727272728, - "map@1": 0.524, - "map@5": 0.6116833333333334, - "map@10": 0.6221575396825396, - "mrr@1": 0.524, - "mrr@5": 0.6116833333333334, - "mrr@10": 0.6221575396825396, + "hit@1": 0.521, + "hit@5": 0.748, + "hit@10": 0.823, + "ndcg_linear@1": 0.521, + "ndcg_linear@5": 0.6452599050997464, + "ndcg_linear@10": 0.6697366881973243, + "ndcg_exponential@1": 0.521, + "ndcg_exponential@5": 0.6452599050997464, + "ndcg_exponential@10": 0.6697366881973243, + "precision@1": 0.521, + "precision@5": 0.14959999999999998, + "precision@10": 0.08230000000000001, + "recall@1": 0.521, + "recall@5": 0.748, + "recall@10": 0.823, + "f1@1": 0.521, + "f1@5": 0.2493333333333334, + "f1@10": 0.14963636363636365, + "map@1": 0.521, + "map@5": 0.6107666666666667, + "map@10": 0.6209880952380952, + "mrr@1": 0.521, + "mrr@5": 0.6107666666666667, + "mrr@10": 0.6209880952380952, "num_pred": 1000, "num_data": 1000 }, "Breakfast": { "hit@1": 0.13625866050808313, - "hit@5": 0.5935334872979214, + "hit@5": 0.5866050808314087, "hit@10": 1.0, "ndcg_linear@1": 0.13625866050808313, - "ndcg_linear@5": 0.3556714764778745, - "ndcg_linear@10": 0.4859126067666473, + "ndcg_linear@5": 0.35349600034287565, + "ndcg_linear@10": 0.4860441595348414, "ndcg_exponential@1": 0.13625866050808313, - "ndcg_exponential@5": 0.3556714764778745, - "ndcg_exponential@10": 0.4859126067666473, + "ndcg_exponential@5": 0.35349600034287565, + "ndcg_exponential@10": 0.4860441595348414, "precision@1": 0.13625866050808313, - "precision@5": 0.11870669745958429, + "precision@5": 0.11732101616628175, "precision@10": 0.1, "recall@1": 0.13625866050808313, - "recall@5": 0.5935334872979214, + "recall@5": 0.5866050808314087, "recall@10": 1.0, "f1@1": 0.13625866050808313, - "f1@5": 0.19784449576597388, + "f1@5": 0.19553502694380298, "f1@10": 0.1818181818181818, "map@1": 0.13625866050808313, - "map@5": 0.27886836027713624, - "map@10": 0.3318990798782947, + "map@5": 0.27809853733641265, + "map@10": 0.3321226951134572, "mrr@1": 0.13625866050808313, - "mrr@5": 0.27886836027713624, - "mrr@10": 0.3318990798782947, + "mrr@5": 0.27809853733641265, + "mrr@10": 0.3321226951134572, "num_pred": 433, "num_data": 433 }, "MVBench": { - "hit@1": 0.3745, + "hit@1": 0.37575, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.3745, - "ndcg_linear@5": 0.7155447146201682, - "ndcg_linear@10": 0.7155447146201682, - "ndcg_exponential@1": 0.3745, - "ndcg_exponential@5": 0.7155447146201682, - "ndcg_exponential@10": 0.7155447146201682, - "precision@1": 0.3745, + "ndcg_linear@1": 0.37575, + "ndcg_linear@5": 0.7158712641094914, + "ndcg_linear@10": 0.7158712641094914, + "ndcg_exponential@1": 0.37575, + "ndcg_exponential@5": 0.7158712641094914, + "ndcg_exponential@10": 0.7158712641094914, + "precision@1": 0.37575, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.3745, + "recall@1": 0.37575, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.3745, + "f1@1": 0.37575, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.3745, - "map@5": 0.6196291666666666, - "map@10": 0.6196291666666666, - "mrr@1": 0.3745, - "mrr@5": 0.6196291666666666, - "mrr@10": 0.6196291666666666, + "map@1": 0.37575, + "map@5": 0.6200875000000001, + "map@10": 0.6200875000000001, + "mrr@1": 0.37575, + "mrr@5": 0.6200875000000001, + "mrr@10": 0.6200875000000001, "num_pred": 4000, "num_data": 4000 }, "Video-MME": { - "hit@1": 0.3425925925925926, + "hit@1": 0.3396296296296296, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.3425925925925926, - "ndcg_linear@5": 0.6913650713075382, - "ndcg_linear@10": 0.6913650713075382, - "ndcg_exponential@1": 0.3425925925925926, - "ndcg_exponential@5": 0.6913650713075382, - "ndcg_exponential@10": 0.6913650713075382, - "precision@1": 0.3425925925925926, + "ndcg_linear@1": 0.3396296296296296, + "ndcg_linear@5": 0.6896382691230423, + "ndcg_linear@10": 0.6896382691230423, + "ndcg_exponential@1": 0.3396296296296296, + "ndcg_exponential@5": 0.6896382691230423, + "ndcg_exponential@10": 0.6896382691230423, + "precision@1": 0.3396296296296296, "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, - "recall@1": 0.3425925925925926, + "recall@1": 0.3396296296296296, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.3425925925925926, + "f1@1": 0.3396296296296296, "f1@5": 0.3333333333333334, "f1@10": 0.18181818181818188, - "map@1": 0.3425925925925926, - "map@5": 0.5881790123456789, - "map@10": 0.5881790123456789, - "mrr@1": 0.3425925925925926, - "mrr@5": 0.5881790123456789, - "mrr@10": 0.5881790123456789, + "map@1": 0.3396296296296296, + "map@5": 0.585895061728395, + "map@10": 0.585895061728395, + "mrr@1": 0.3396296296296296, + "mrr@5": 0.585895061728395, + "mrr@10": 0.585895061728395, "num_pred": 2700, "num_data": 2700 }, "NExTQA": { - "hit@1": 0.3949089210649229, + "hit@1": 0.3939747781410556, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.3949089210649229, - "ndcg_linear@5": 0.7033624022532277, - "ndcg_linear@10": 0.7033624022532277, - "ndcg_exponential@1": 0.3949089210649229, - "ndcg_exponential@5": 0.7033624022532277, - "ndcg_exponential@10": 0.7033624022532277, - "precision@1": 0.3949089210649229, + "ndcg_linear@1": 0.3939747781410556, + "ndcg_linear@5": 0.7029264712441474, + "ndcg_linear@10": 0.7029264712441474, + "ndcg_exponential@1": 0.3939747781410556, + "ndcg_exponential@5": 0.7029264712441474, + "ndcg_exponential@10": 0.7029264712441474, + "precision@1": 0.3939747781410556, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.3949089210649229, + "recall@1": 0.3939747781410556, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.3949089210649229, + "f1@1": 0.3939747781410556, "f1@5": 0.3333333333333333, "f1@10": 0.1818181818181818, - "map@1": 0.3949089210649229, - "map@5": 0.605927915304375, - "map@10": 0.605927915304375, - "mrr@1": 0.3949089210649229, - "mrr@5": 0.6057527635061498, - "mrr@10": 0.6057527635061498, + "map@1": 0.3939747781410556, + "map@5": 0.6053421298458664, + "map@10": 0.6053421298458664, + "mrr@1": 0.3939747781410556, + "mrr@5": 0.6051669780476413, + "mrr@10": 0.6051669780476413, "num_pred": 8564, "num_data": 8564 }, "EgoSchema": { - "hit@1": 0.408, + "hit@1": 0.406, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.408, - "ndcg_linear@5": 0.7172123608985213, - "ndcg_linear@10": 0.7172123608985213, - "ndcg_exponential@1": 0.408, - "ndcg_exponential@5": 0.7172123608985213, - "ndcg_exponential@10": 0.7172123608985213, - "precision@1": 0.408, + "ndcg_linear@1": 0.406, + "ndcg_linear@5": 0.7149055939675113, + "ndcg_linear@10": 0.7149055939675113, + "ndcg_exponential@1": 0.406, + "ndcg_exponential@5": 0.7149055939675113, + "ndcg_exponential@10": 0.7149055939675113, + "precision@1": 0.406, "precision@5": 0.2, "precision@10": 0.1, - "recall@1": 0.408, + "recall@1": 0.406, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.408, + "f1@1": 0.406, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.408, - "map@5": 0.6235, - "map@10": 0.6235, - "mrr@1": 0.408, - "mrr@5": 0.6235, - "mrr@10": 0.6235, + "map@1": 0.406, + "map@5": 0.6206333333333333, + "map@10": 0.6206333333333333, + "mrr@1": 0.406, + "mrr@5": 0.6206333333333333, + "mrr@10": 0.6206333333333333, "num_pred": 500, "num_data": 500 }, "ActivityNetQA": { - "hit@1": 0.58, + "hit@1": 0.572, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.58, - "ndcg_linear@5": 0.8449904965000122, - "ndcg_linear@10": 0.8449904965000122, - "ndcg_exponential@1": 0.58, - "ndcg_exponential@5": 0.8449904965000122, - "ndcg_exponential@10": 0.8449904965000122, - "precision@1": 0.58, + "ndcg_linear@1": 0.572, + "ndcg_linear@5": 0.842037934528584, + "ndcg_linear@10": 0.842037934528584, + "ndcg_exponential@1": 0.572, + "ndcg_exponential@5": 0.842037934528584, + "ndcg_exponential@10": 0.842037934528584, + "precision@1": 0.572, "precision@5": 0.20000000000000004, "precision@10": 0.10000000000000002, - "recall@1": 0.58, + "recall@1": 0.572, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.58, + "f1@1": 0.572, "f1@5": 0.3333333333333333, "f1@10": 0.18181818181818182, - "map@1": 0.58, - "map@5": 0.79, - "map@10": 0.79, - "mrr@1": 0.58, - "mrr@5": 0.79, - "mrr@10": 0.79, + "map@1": 0.572, + "map@5": 0.786, + "map@10": 0.786, + "mrr@1": 0.572, + "mrr@5": 0.786, + "mrr@10": 0.786, "num_pred": 1000, "num_data": 1000 }, "DiDeMo": { - "hit@1": 0.2201195219123506, + "hit@1": 0.2151394422310757, "hit@5": 0.41733067729083667, - "hit@10": 0.5119521912350598, - "ndcg_linear@1": 0.2201195219123506, - "ndcg_linear@5": 0.32478232828071807, - "ndcg_linear@10": 0.355304006642377, - "ndcg_exponential@1": 0.2201195219123506, - "ndcg_exponential@5": 0.32478232828071807, - "ndcg_exponential@10": 0.355304006642377, - "precision@1": 0.2201195219123506, + "hit@10": 0.5199203187250996, + "ndcg_linear@1": 0.2151394422310757, + "ndcg_linear@5": 0.3232929829377131, + "ndcg_linear@10": 0.3563601578277361, + "ndcg_exponential@1": 0.2151394422310757, + "ndcg_exponential@5": 0.3232929829377131, + "ndcg_exponential@10": 0.3563601578277361, + "precision@1": 0.2151394422310757, "precision@5": 0.08346613545816733, - "precision@10": 0.051195219123505976, - "recall@1": 0.2201195219123506, + "precision@10": 0.05199203187250996, + "recall@1": 0.2151394422310757, "recall@5": 0.41733067729083667, - "recall@10": 0.5119521912350598, - "f1@1": 0.2201195219123506, + "recall@10": 0.5199203187250996, + "f1@1": 0.2151394422310757, "f1@5": 0.13911022576361223, - "f1@10": 0.0930822165881927, - "map@1": 0.2201195219123506, - "map@5": 0.2940073041168659, - "map@10": 0.30655315246948717, - "mrr@1": 0.2201195219123506, - "mrr@5": 0.2940073041168659, - "mrr@10": 0.30655315246948717, + "f1@10": 0.09453096704092723, + "map@1": 0.2151394422310757, + "map@5": 0.29193227091633467, + "map@10": 0.3055136596471258, + "mrr@1": 0.2151394422310757, + "mrr@5": 0.29193227091633467, + "mrr@10": 0.3055136596471258, "num_pred": 1004, "num_data": 1004 }, "MSR-VTT": { - "hit@1": 0.273, - "hit@5": 0.498, - "hit@10": 0.601, - "ndcg_linear@1": 0.273, - "ndcg_linear@5": 0.39500395524644993, - "ndcg_linear@10": 0.4288322570956004, - "ndcg_exponential@1": 0.273, - "ndcg_exponential@5": 0.39500395524644993, - "ndcg_exponential@10": 0.4288322570956004, - "precision@1": 0.273, - "precision@5": 0.0996, - "precision@10": 0.060099999999999994, - "recall@1": 0.273, - "recall@5": 0.498, - "recall@10": 0.601, - "f1@1": 0.273, - "f1@5": 0.16600000000000006, - "f1@10": 0.1092727272727273, - "map@1": 0.273, - "map@5": 0.3605333333333333, - "map@10": 0.3747972222222222, - "mrr@1": 0.273, - "mrr@5": 0.3605333333333333, - "mrr@10": 0.3747972222222222, + "hit@1": 0.27, + "hit@5": 0.5, + "hit@10": 0.594, + "ndcg_linear@1": 0.27, + "ndcg_linear@5": 0.394838027994707, + "ndcg_linear@10": 0.42580939676509716, + "ndcg_exponential@1": 0.27, + "ndcg_exponential@5": 0.394838027994707, + "ndcg_exponential@10": 0.42580939676509716, + "precision@1": 0.27, + "precision@5": 0.1, + "precision@10": 0.059399999999999994, + "recall@1": 0.27, + "recall@5": 0.5, + "recall@10": 0.594, + "f1@1": 0.27, + "f1@5": 0.16666666666666669, + "f1@10": 0.10800000000000003, + "map@1": 0.27, + "map@5": 0.3596333333333333, + "map@10": 0.37274841269841275, + "mrr@1": 0.27, + "mrr@5": 0.3596333333333333, + "mrr@10": 0.37274841269841275, "num_pred": 1000, "num_data": 1000 }, "MSVD": { - "hit@1": 0.4761194029850746, + "hit@1": 0.47313432835820896, "hit@5": 0.753731343283582, - "hit@10": 0.8223880597014925, - "ndcg_linear@1": 0.4761194029850746, - "ndcg_linear@5": 0.6277485828345324, - "ndcg_linear@10": 0.6502611190764599, - "ndcg_exponential@1": 0.4761194029850746, - "ndcg_exponential@5": 0.6277485828345324, - "ndcg_exponential@10": 0.6502611190764599, - "precision@1": 0.4761194029850746, - "precision@5": 0.15074626865671642, - "precision@10": 0.08223880597014925, - "recall@1": 0.4761194029850746, + "hit@10": 0.826865671641791, + "ndcg_linear@1": 0.47313432835820896, + "ndcg_linear@5": 0.6261676007279617, + "ndcg_linear@10": 0.6499175705151878, + "ndcg_exponential@1": 0.47313432835820896, + "ndcg_exponential@5": 0.6261676007279617, + "ndcg_exponential@10": 0.6499175705151878, + "precision@1": 0.47313432835820896, + "precision@5": 0.1507462686567164, + "precision@10": 0.08268656716417909, + "recall@1": 0.47313432835820896, "recall@5": 0.753731343283582, - "recall@10": 0.8223880597014925, - "f1@1": 0.4761194029850746, + "recall@10": 0.826865671641791, + "f1@1": 0.47313432835820896, "f1@5": 0.25124378109452744, - "f1@10": 0.14952510176390776, - "map@1": 0.4761194029850746, - "map@5": 0.5854726368159203, - "map@10": 0.5949484719260838, - "mrr@1": 0.4761194029850746, - "mrr@5": 0.5854726368159203, - "mrr@10": 0.5949484719260838, + "f1@10": 0.1503392130257802, + "map@1": 0.47313432835820896, + "map@5": 0.583407960199005, + "map@10": 0.5932699597251836, + "mrr@1": 0.47313432835820896, + "mrr@5": 0.583407960199005, + "mrr@10": 0.5932699597251836, "num_pred": 670, "num_data": 670 }, "VATEX": { - "hit@1": 0.22956677087985708, - "hit@5": 0.44796784278695845, - "hit@10": 0.5482358195623046, - "ndcg_linear@1": 0.22956677087985708, - "ndcg_linear@5": 0.3442450550592858, - "ndcg_linear@10": 0.37658573093046976, - "ndcg_exponential@1": 0.22956677087985708, - "ndcg_exponential@5": 0.3442450550592858, - "ndcg_exponential@10": 0.37658573093046976, - "precision@1": 0.22956677087985708, - "precision@5": 0.0895935685573917, - "precision@10": 0.05482358195623046, - "recall@1": 0.22956677087985708, - "recall@5": 0.44796784278695845, - "recall@10": 0.5482358195623046, - "f1@1": 0.22956677087985708, - "f1@5": 0.1493226142623195, - "f1@10": 0.09967923992041902, - "map@1": 0.22956677087985708, - "map@5": 0.3098518683936281, - "map@10": 0.3231518109700334, - "mrr@1": 0.22956677087985708, - "mrr@5": 0.3098518683936281, - "mrr@10": 0.3231518109700334, + "hit@1": 0.23046002679767755, + "hit@5": 0.44707458686913804, + "hit@10": 0.5444394819115677, + "ndcg_linear@1": 0.23046002679767755, + "ndcg_linear@5": 0.3439938223438492, + "ndcg_linear@10": 0.3755043733402501, + "ndcg_exponential@1": 0.23046002679767755, + "ndcg_exponential@5": 0.3439938223438492, + "ndcg_exponential@10": 0.3755043733402501, + "precision@1": 0.23046002679767755, + "precision@5": 0.08941491737382759, + "precision@10": 0.05444394819115677, + "recall@1": 0.23046002679767755, + "recall@5": 0.44707458686913804, + "recall@10": 0.5444394819115677, + "f1@1": 0.23046002679767755, + "f1@5": 0.1490248622897127, + "f1@10": 0.09898899671119411, + "map@1": 0.23046002679767755, + "map@5": 0.3098369807949977, + "map@10": 0.32285441346406063, + "mrr@1": 0.23046002679767755, + "mrr@5": 0.3098369807949977, + "mrr@10": 0.32285441346406063, "num_pred": 4478, "num_data": 4478 }, "YouCook2": { "hit@1": 0.07895564642969487, - "hit@5": 0.18402013211701793, - "hit@10": 0.2491349480968858, + "hit@5": 0.18559295375904372, + "hit@10": 0.24724756212645485, "ndcg_linear@1": 0.07895564642969487, - "ndcg_linear@5": 0.13201194607861527, - "ndcg_linear@10": 0.1529921311761448, + "ndcg_linear@5": 0.13275976893978403, + "ndcg_linear@10": 0.15261892582807532, "ndcg_exponential@1": 0.07895564642969487, - "ndcg_exponential@5": 0.13201194607861527, - "ndcg_exponential@10": 0.1529921311761448, + "ndcg_exponential@5": 0.13275976893978403, + "ndcg_exponential@10": 0.15261892582807532, "precision@1": 0.07895564642969487, - "precision@5": 0.03680402642340359, - "precision@10": 0.02491349480968858, + "precision@5": 0.03711859075180875, + "precision@10": 0.024724756212645485, "recall@1": 0.07895564642969487, - "recall@5": 0.18402013211701793, - "recall@10": 0.2491349480968858, + "recall@5": 0.18559295375904372, + "recall@10": 0.24724756212645485, "f1@1": 0.07895564642969487, - "f1@5": 0.061340044039005975, - "f1@10": 0.04529726329034287, + "f1@5": 0.06186431791968125, + "f1@10": 0.04495410220480997, "map@1": 0.07895564642969487, - "map@5": 0.11498374750969906, - "map@10": 0.12359781702342257, + "map@5": 0.11545035126350005, + "map@10": 0.12360393355203043, "mrr@1": 0.07895564642969487, - "mrr@5": 0.11498374750969906, - "mrr@10": 0.12359781702342257, + "mrr@5": 0.11545035126350005, + "mrr@10": 0.12360393355203043, "num_pred": 3179, "num_data": 3179 }, "QVHighlight": { - "hit@1": 0.4358264081255771, - "hit@5": 0.8116343490304709, + "hit@1": 0.43951985226223456, + "hit@5": 0.8097876269621422, "hit@10": 1.0, - "ndcg_linear@1": 0.4358264081255771, - "ndcg_linear@5": 0.6322706491755596, - "ndcg_linear@10": 0.6929622320450238, - "ndcg_exponential@1": 0.4358264081255771, - "ndcg_exponential@5": 0.6322706491755596, - "ndcg_exponential@10": 0.6929622320450238, - "precision@1": 0.4358264081255771, - "precision@5": 0.1623268698060942, + "ndcg_linear@1": 0.43951985226223456, + "ndcg_linear@5": 0.6339643053190146, + "ndcg_linear@10": 0.6953919675747608, + "ndcg_exponential@1": 0.43951985226223456, + "ndcg_exponential@5": 0.6339643053190146, + "ndcg_exponential@10": 0.6953919675747608, + "precision@1": 0.43951985226223456, + "precision@5": 0.16195752539242841, "precision@10": 0.09999999999999998, - "recall@1": 0.4358264081255771, - "recall@5": 0.8116343490304709, + "recall@1": 0.43951985226223456, + "recall@5": 0.8097876269621422, "recall@10": 1.0, - "f1@1": 0.4358264081255771, - "f1@5": 0.270544783010157, + "f1@1": 0.43951985226223456, + "f1@5": 0.26992920898738076, "f1@10": 0.18181818181818185, - "map@1": 0.4358264081255771, - "map@5": 0.5728685749461373, - "map@10": 0.5977850327573319, - "mrr@1": 0.4358264081255771, - "mrr@5": 0.5728685749461373, - "mrr@10": 0.5977850327573319, + "map@1": 0.43951985226223456, + "map@5": 0.5756694367497691, + "map@10": 0.6009764909349398, + "mrr@1": 0.43951985226223456, + "mrr@5": 0.5756694367497691, + "mrr@10": 0.6009764909349398, "num_pred": 1083, "num_data": 1083 }, "Charades-STA": { - "hit@1": 0.1485557083906465, - "hit@5": 0.5281980742778541, + "hit@1": 0.14305364511691884, + "hit@5": 0.5240715268225584, "hit@10": 1.0, - "ndcg_linear@1": 0.1485557083906465, - "ndcg_linear@5": 0.336037308583379, - "ndcg_linear@10": 0.4847984333087624, - "ndcg_exponential@1": 0.1485557083906465, - "ndcg_exponential@5": 0.336037308583379, - "ndcg_exponential@10": 0.4847984333087624, - "precision@1": 0.1485557083906465, - "precision@5": 0.10563961485557083, + "ndcg_linear@1": 0.14305364511691884, + "ndcg_linear@5": 0.3333352341971968, + "ndcg_linear@10": 0.48335766594478874, + "ndcg_exponential@1": 0.14305364511691884, + "ndcg_exponential@5": 0.3333352341971968, + "ndcg_exponential@10": 0.48335766594478874, + "precision@1": 0.14305364511691884, + "precision@5": 0.10481430536451168, "precision@10": 0.09999999999999998, - "recall@1": 0.1485557083906465, - "recall@5": 0.5281980742778541, + "recall@1": 0.14305364511691884, + "recall@5": 0.5240715268225584, "recall@10": 1.0, - "f1@1": 0.1485557083906465, - "f1@5": 0.17606602475928476, + "f1@1": 0.14305364511691884, + "f1@5": 0.17469050894085283, "f1@10": 0.18181818181818185, - "map@1": 0.1485557083906465, - "map@5": 0.27345254470426406, - "map@10": 0.33267013820658936, - "mrr@1": 0.1485557083906465, - "mrr@5": 0.27345254470426406, - "mrr@10": 0.33267013820658936, + "map@1": 0.14305364511691884, + "map@5": 0.2710683172856488, + "map@10": 0.330753695334163, + "mrr@1": 0.14305364511691884, + "mrr@5": 0.2710683172856488, + "mrr@10": 0.330753695334163, "num_pred": 727, "num_data": 727 }, "MomentSeeker": { - "hit@1": 0.3477777777777778, - "hit@5": 0.8511111111111112, + "hit@1": 0.3714107365792759, + "hit@5": 0.8558052434456929, + "hit@10": 1.0, + "ndcg_linear@1": 0.3714107365792759, + "ndcg_linear@5": 0.620703086971645, + "ndcg_linear@10": 0.6766798473590683, + "ndcg_exponential@1": 0.3714107365792759, + "ndcg_exponential@5": 0.620703086971645, + "ndcg_exponential@10": 0.6766798473590683, + "precision@1": 0.3714107365792759, + "precision@5": 0.18776529338327091, + "precision@10": 0.1146067415730337, + "recall@1": 0.3450374531835206, + "recall@5": 0.8345297544735746, + "recall@10": 0.9989596337910945, + "f1@1": 0.3527475576164714, + "f1@5": 0.30129944268521053, + "f1@10": 0.20267599996064414, + "map@1": 0.3714107365792759, + "map@5": 0.5427224649743376, + "map@10": 0.5690497245506608, + "mrr@1": 0.3714107365792759, + "mrr@5": 0.5567831876820641, + "mrr@10": 0.5764557695737471, + "num_pred": 1602, + "num_data": 1602 + }, + "MomentSeeker_1k8": { + "hit@1": 0.3472222222222222, + "hit@5": 0.8477777777777777, "hit@10": 1.0, - "ndcg_linear@1": 0.3477777777777778, - "ndcg_linear@5": 0.6067492115086224, - "ndcg_linear@10": 0.6630917380371577, - "ndcg_exponential@1": 0.3477777777777778, - "ndcg_exponential@5": 0.6067492115086224, - "ndcg_exponential@10": 0.6630917380371577, - "precision@1": 0.3477777777777778, - "precision@5": 0.18533333333333335, + "ndcg_linear@1": 0.3472222222222222, + "ndcg_linear@5": 0.6047084133205788, + "ndcg_linear@10": 0.6623359049333758, + "ndcg_exponential@1": 0.3472222222222222, + "ndcg_exponential@5": 0.6047084133205788, + "ndcg_exponential@10": 0.6623359049333758, + "precision@1": 0.3472222222222222, + "precision@5": 0.18444444444444444, "precision@10": 0.11305555555555558, - "recall@1": 0.32458333333333333, - "recall@5": 0.8322685185185186, + "recall@1": 0.32375, + "recall@5": 0.8288425925925925, "recall@10": 0.9990740740740741, - "f1@1": 0.33135273368606705, - "f1@5": 0.29835491668825004, + "f1@1": 0.3306119929453263, + "f1@5": 0.2969924828258162, "f1@10": 0.20046581504914834, - "map@1": 0.3477777777777778, - "map@5": 0.5258817901234568, - "map@10": 0.5519251543209877, - "mrr@1": 0.3477777777777778, - "mrr@5": 0.5379629629629631, - "mrr@10": 0.5581768077601411, + "map@1": 0.3472222222222222, + "map@5": 0.5241711419753087, + "map@10": 0.5508729056437389, + "mrr@1": 0.3472222222222222, + "mrr@5": 0.5366388888888889, + "mrr@10": 0.5574179894179894, "num_pred": 1800, "num_data": 1800 } }, "visdoc": { "ViDoRe_arxivqa": { - "hit@1": 0.774, - "hit@5": 0.878, - "hit@10": 0.91, - "ndcg_linear@1": 0.774, - "ndcg_linear@5": 0.8276032117095806, - "ndcg_linear@10": 0.8379559054452919, - "ndcg_exponential@1": 0.774, - "ndcg_exponential@5": 0.8276032117095806, - "ndcg_exponential@10": 0.8379559054452919, - "precision@1": 0.774, - "precision@5": 0.1756, - "precision@10": 0.091, - "recall@1": 0.774, - "recall@5": 0.878, - "recall@10": 0.91, - "f1@1": 0.774, - "f1@5": 0.2926666666666667, - "f1@10": 0.16545454545454544, - "map@1": 0.774, - "map@5": 0.811, - "map@10": 0.8152769841269842, - "mrr@1": 0.774, - "mrr@5": 0.811, - "mrr@10": 0.8152769841269842, + "hit@1": 0.776, + "hit@5": 0.866, + "hit@10": 0.914, + "ndcg_linear@1": 0.776, + "ndcg_linear@5": 0.8253367092012197, + "ndcg_linear@10": 0.8408466929159699, + "ndcg_exponential@1": 0.776, + "ndcg_exponential@5": 0.8253367092012197, + "ndcg_exponential@10": 0.8408466929159699, + "precision@1": 0.776, + "precision@5": 0.1732, + "precision@10": 0.09140000000000001, + "recall@1": 0.776, + "recall@5": 0.866, + "recall@10": 0.914, + "f1@1": 0.776, + "f1@5": 0.28866666666666674, + "f1@10": 0.16618181818181818, + "map@1": 0.776, + "map@5": 0.8116666666666668, + "map@10": 0.8180650793650793, + "mrr@1": 0.776, + "mrr@5": 0.8116666666666668, + "mrr@10": 0.8180650793650793, "num_pred": 500, "num_data": 500 }, "ViDoRe_docvqa": { - "hit@1": 0.4434589800443459, - "hit@5": 0.6075388026607539, - "hit@10": 0.6651884700665188, - "ndcg_linear@1": 0.4434589800443459, - "ndcg_linear@5": 0.5311135724096901, - "ndcg_linear@10": 0.549126558201007, - "ndcg_exponential@1": 0.4434589800443459, - "ndcg_exponential@5": 0.5311135724096901, - "ndcg_exponential@10": 0.549126558201007, - "precision@1": 0.4434589800443459, - "precision@5": 0.12416851441241683, - "precision@10": 0.06829268292682926, - "recall@1": 0.437250554323725, - "recall@5": 0.6023281596452328, - "recall@10": 0.6581116038433111, - "f1@1": 0.43902439024390244, - "f1@5": 0.20455818691112812, - "f1@10": 0.12296562762194693, - "map@1": 0.4434589800443459, - "map@5": 0.505469327420547, - "map@10": 0.512853601344455, - "mrr@1": 0.4434589800443459, - "mrr@5": 0.5099408721359941, - "mrr@10": 0.5174251223031711, + "hit@1": 0.4611973392461197, + "hit@5": 0.6341463414634146, + "hit@10": 0.6873614190687362, + "ndcg_linear@1": 0.4611973392461197, + "ndcg_linear@5": 0.5516050966468681, + "ndcg_linear@10": 0.5691769530394468, + "ndcg_exponential@1": 0.4611973392461197, + "ndcg_exponential@5": 0.5516050966468681, + "ndcg_exponential@10": 0.5691769530394468, + "precision@1": 0.4611973392461197, + "precision@5": 0.12949002217294897, + "precision@10": 0.07073170731707316, + "recall@1": 0.4575018477457502, + "recall@5": 0.6289356984478935, + "recall@10": 0.6822246858832225, + "f1@1": 0.458610495195861, + "f1@5": 0.21342736651201505, + "f1@10": 0.1274898050951266, + "map@1": 0.4611973392461197, + "map@5": 0.5244641537324464, + "map@10": 0.5320003167564143, + "mrr@1": 0.4611973392461197, + "mrr@5": 0.5275314116777531, + "mrr@10": 0.5346443529370358, "num_pred": 451, "num_data": 451 }, "ViDoRe_infovqa": { - "hit@1": 0.8481781376518218, - "hit@5": 0.9453441295546559, - "hit@10": 0.9615384615384616, - "ndcg_linear@1": 0.8481781376518218, - "ndcg_linear@5": 0.9016308878511893, - "ndcg_linear@10": 0.9073181214772046, - "ndcg_exponential@1": 0.8481781376518218, - "ndcg_exponential@5": 0.9016308878511893, - "ndcg_exponential@10": 0.9073181214772046, - "precision@1": 0.8481781376518218, - "precision@5": 0.1890688259109312, - "precision@10": 0.09635627530364373, - "recall@1": 0.8481781376518218, - "recall@5": 0.9427125506072874, - "recall@10": 0.9599190283400809, - "f1@1": 0.8481781376518218, - "f1@5": 0.3147484094852516, - "f1@10": 0.17503373819163293, - "map@1": 0.8481781376518218, - "map@5": 0.8873481781376519, - "map@10": 0.8898359681254419, - "mrr@1": 0.8481781376518218, - "mrr@5": 0.8883603238866398, - "mrr@10": 0.8905589293747189, + "hit@1": 0.8623481781376519, + "hit@5": 0.9433198380566802, + "hit@10": 0.9696356275303644, + "ndcg_linear@1": 0.8623481781376519, + "ndcg_linear@5": 0.9069263083568274, + "ndcg_linear@10": 0.916219744007854, + "ndcg_exponential@1": 0.8623481781376519, + "ndcg_exponential@5": 0.9069263083568274, + "ndcg_exponential@10": 0.916219744007854, + "precision@1": 0.8623481781376519, + "precision@5": 0.18866396761133603, + "precision@10": 0.09716599190283401, + "recall@1": 0.8607287449392712, + "recall@5": 0.9406882591093118, + "recall@10": 0.9680161943319838, + "f1@1": 0.8609986504723348, + "f1@5": 0.31407364565259305, + "f1@10": 0.17650595019016072, + "map@1": 0.8623481781376519, + "map@5": 0.8949224021592443, + "map@10": 0.8990754128912024, + "mrr@1": 0.8623481781376519, + "mrr@5": 0.8967948717948717, + "mrr@10": 0.900694846089583, "num_pred": 494, "num_data": 494 }, "ViDoRe_tabfquad": { - "hit@1": 0.8892857142857142, - "hit@5": 0.9678571428571429, - "hit@10": 0.975, - "ndcg_linear@1": 0.8892857142857142, - "ndcg_linear@5": 0.9332834391651387, - "ndcg_linear@10": 0.9356005756298494, - "ndcg_exponential@1": 0.8892857142857142, - "ndcg_exponential@5": 0.9332834391651387, - "ndcg_exponential@10": 0.9356005756298494, - "precision@1": 0.8892857142857142, - "precision@5": 0.19357142857142853, - "precision@10": 0.09749999999999999, - "recall@1": 0.8892857142857142, - "recall@5": 0.9678571428571429, - "recall@10": 0.975, - "f1@1": 0.8892857142857142, - "f1@5": 0.3226190476190477, - "f1@10": 0.1772727272727273, - "map@1": 0.8892857142857142, - "map@5": 0.921547619047619, - "map@10": 0.9225042517006803, - "mrr@1": 0.8892857142857142, - "mrr@5": 0.921547619047619, - "mrr@10": 0.9225042517006803, + "hit@1": 0.9, + "hit@5": 0.9642857142857143, + "hit@10": 0.9785714285714285, + "ndcg_linear@1": 0.9, + "ndcg_linear@5": 0.93305889527537, + "ndcg_linear@10": 0.9373892273526313, + "ndcg_exponential@1": 0.9, + "ndcg_exponential@5": 0.93305889527537, + "ndcg_exponential@10": 0.9373892273526313, + "precision@1": 0.9, + "precision@5": 0.19285714285714284, + "precision@10": 0.09785714285714284, + "recall@1": 0.9, + "recall@5": 0.9642857142857143, + "recall@10": 0.9785714285714285, + "f1@1": 0.9, + "f1@5": 0.3214285714285715, + "f1@10": 0.17792207792207795, + "map@1": 0.9, + "map@5": 0.9227380952380952, + "map@10": 0.924359410430839, + "mrr@1": 0.9, + "mrr@5": 0.9227380952380952, + "mrr@10": 0.924359410430839, "num_pred": 280, "num_data": 280 }, "ViDoRe_tatdqa": { - "hit@1": 0.5577156743620899, - "hit@5": 0.8250303766707169, - "hit@10": 0.8894289185905225, - "ndcg_linear@1": 0.5577156743620899, - "ndcg_linear@5": 0.6993890766047229, - "ndcg_linear@10": 0.7206656381690775, - "ndcg_exponential@1": 0.5577156743620899, - "ndcg_exponential@5": 0.6993890766047229, - "ndcg_exponential@10": 0.7206656381690775, - "precision@1": 0.5577156743620899, - "precision@5": 0.16537059538274604, - "precision@10": 0.08924665856622113, - "recall@1": 0.5562575941676793, - "recall@5": 0.8235722964763061, - "recall@10": 0.8885783718104495, - "f1@1": 0.556611068154203, - "f1@5": 0.2752357808250883, - "f1@10": 0.1621156890901727, - "map@1": 0.5577156743620899, - "map@5": 0.6577612393681652, - "map@10": 0.6667486354606647, - "mrr@1": 0.5577156743620899, - "mrr@5": 0.6586776022681248, - "mrr@10": 0.6675377056452391, + "hit@1": 0.5650060753341434, + "hit@5": 0.8238153098420413, + "hit@10": 0.8985419198055893, + "ndcg_linear@1": 0.5650060753341434, + "ndcg_linear@5": 0.7025977138647097, + "ndcg_linear@10": 0.7268099542086717, + "ndcg_exponential@1": 0.5650060753341434, + "ndcg_exponential@5": 0.7025977138647097, + "ndcg_exponential@10": 0.7268099542086717, + "precision@1": 0.5650060753341434, + "precision@5": 0.16512758201701092, + "precision@10": 0.09009720534629405, + "recall@1": 0.5637910085054678, + "recall@5": 0.8229040097205347, + "recall@10": 0.8973876063183475, + "f1@1": 0.564196030781693, + "f1@5": 0.2749522652317307, + "f1@10": 0.16367134283294676, + "map@1": 0.5650060753341434, + "map@5": 0.6622873633049817, + "map@10": 0.6723562653088776, + "mrr@1": 0.5650060753341434, + "mrr@5": 0.662910085054678, + "mrr@10": 0.6729384848309514, "num_pred": 1646, "num_data": 1646 }, "ViDoRe_shiftproject": { - "hit@1": 0.8, - "hit@5": 0.97, + "hit@1": 0.86, + "hit@5": 0.98, "hit@10": 0.99, - "ndcg_linear@1": 0.8, - "ndcg_linear@5": 0.894703974563633, - "ndcg_linear@10": 0.9010476078536062, - "ndcg_exponential@1": 0.8, - "ndcg_exponential@5": 0.894703974563633, - "ndcg_exponential@10": 0.9010476078536062, - "precision@1": 0.8, - "precision@5": 0.19399999999999998, + "ndcg_linear@1": 0.86, + "ndcg_linear@5": 0.9284718483307358, + "ndcg_linear@10": 0.9318051816640692, + "ndcg_exponential@1": 0.86, + "ndcg_exponential@5": 0.9284718483307358, + "ndcg_exponential@10": 0.9318051816640692, + "precision@1": 0.86, + "precision@5": 0.19599999999999995, "precision@10": 0.09899999999999999, - "recall@1": 0.8, - "recall@5": 0.97, + "recall@1": 0.86, + "recall@5": 0.98, "recall@10": 0.99, - "f1@1": 0.8, - "f1@5": 0.3233333333333334, + "f1@1": 0.86, + "f1@5": 0.32666666666666677, "f1@10": 0.18000000000000008, - "map@1": 0.8, - "map@5": 0.8691666666666665, - "map@10": 0.8717063492063492, - "mrr@1": 0.8, - "mrr@5": 0.8691666666666665, - "mrr@10": 0.8717063492063492, + "map@1": 0.86, + "map@5": 0.9108333333333334, + "map@10": 0.9122619047619048, + "mrr@1": 0.86, + "mrr@5": 0.9108333333333334, + "mrr@10": 0.9122619047619048, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_artificial_intelligence": { - "hit@1": 0.96, + "hit@1": 0.97, "hit@5": 0.99, "hit@10": 1.0, - "ndcg_linear@1": 0.96, - "ndcg_linear@5": 0.97517782560806, - "ndcg_linear@10": 0.9787398974791403, - "ndcg_exponential@1": 0.96, - "ndcg_exponential@5": 0.97517782560806, - "ndcg_exponential@10": 0.9787398974791403, - "precision@1": 0.96, + "ndcg_linear@1": 0.97, + "ndcg_linear@5": 0.9813092975357145, + "ndcg_linear@10": 0.9846426308690479, + "ndcg_exponential@1": 0.97, + "ndcg_exponential@5": 0.9813092975357145, + "ndcg_exponential@10": 0.9846426308690479, + "precision@1": 0.97, "precision@5": 0.19799999999999998, "precision@10": 0.09999999999999998, - "recall@1": 0.96, + "recall@1": 0.97, "recall@5": 0.99, "recall@10": 1.0, - "f1@1": 0.96, + "f1@1": 0.97, "f1@5": 0.3300000000000001, "f1@10": 0.1818181818181819, - "map@1": 0.96, - "map@5": 0.9703333333333333, - "map@10": 0.972, - "mrr@1": 0.96, - "mrr@5": 0.9703333333333333, - "mrr@10": 0.972, + "map@1": 0.97, + "map@5": 0.9783333333333333, + "map@10": 0.9797619047619047, + "mrr@1": 0.97, + "mrr@5": 0.9783333333333333, + "mrr@10": 0.9797619047619047, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_energy": { - "hit@1": 0.88, + "hit@1": 0.9, "hit@5": 0.95, - "hit@10": 0.96, - "ndcg_linear@1": 0.88, - "ndcg_linear@5": 0.9191057182152036, - "ndcg_linear@10": 0.9221160181718434, - "ndcg_exponential@1": 0.88, - "ndcg_exponential@5": 0.9191057182152036, - "ndcg_exponential@10": 0.9221160181718434, - "precision@1": 0.88, + "hit@10": 0.97, + "ndcg_linear@1": 0.9, + "ndcg_linear@5": 0.9256160631164485, + "ndcg_linear@10": 0.9316613601474848, + "ndcg_exponential@1": 0.9, + "ndcg_exponential@5": 0.9256160631164485, + "ndcg_exponential@10": 0.9316613601474848, + "precision@1": 0.9, "precision@5": 0.18999999999999997, - "precision@10": 0.09599999999999997, - "recall@1": 0.88, + "precision@10": 0.09699999999999998, + "recall@1": 0.9, "recall@5": 0.95, - "recall@10": 0.96, - "f1@1": 0.88, + "recall@10": 0.97, + "f1@1": 0.9, "f1@5": 0.3166666666666667, - "f1@10": 0.17454545454545464, - "map@1": 0.88, - "map@5": 0.9086666666666667, - "map@10": 0.9097777777777778, - "mrr@1": 0.88, - "mrr@5": 0.9086666666666667, - "mrr@10": 0.9097777777777778, + "f1@10": 0.17636363636363647, + "map@1": 0.9, + "map@5": 0.9175, + "map@10": 0.91975, + "mrr@1": 0.9, + "mrr@5": 0.9175, + "mrr@10": 0.91975, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_government_reports": { - "hit@1": 0.91, - "hit@5": 0.97, - "hit@10": 0.98, - "ndcg_linear@1": 0.91, - "ndcg_linear@5": 0.9458532532593068, - "ndcg_linear@10": 0.9487439015224857, - "ndcg_exponential@1": 0.91, - "ndcg_exponential@5": 0.9458532532593068, - "ndcg_exponential@10": 0.9487439015224857, - "precision@1": 0.91, - "precision@5": 0.19399999999999995, - "precision@10": 0.09799999999999998, - "recall@1": 0.91, - "recall@5": 0.97, - "recall@10": 0.98, - "f1@1": 0.91, - "f1@5": 0.3233333333333334, - "f1@10": 0.17818181818181827, - "map@1": 0.91, - "map@5": 0.9375, - "map@10": 0.9384999999999999, - "mrr@1": 0.91, - "mrr@5": 0.9375, - "mrr@10": 0.9384999999999999, + "hit@1": 0.94, + "hit@5": 0.99, + "hit@10": 1.0, + "ndcg_linear@1": 0.94, + "ndcg_linear@5": 0.9715464876785728, + "ndcg_linear@10": 0.9745567876352126, + "ndcg_exponential@1": 0.94, + "ndcg_exponential@5": 0.9715464876785728, + "ndcg_exponential@10": 0.9745567876352126, + "precision@1": 0.94, + "precision@5": 0.19799999999999998, + "precision@10": 0.09999999999999998, + "recall@1": 0.94, + "recall@5": 0.99, + "recall@10": 1.0, + "f1@1": 0.94, + "f1@5": 0.3300000000000001, + "f1@10": 0.1818181818181819, + "map@1": 0.94, + "map@5": 0.965, + "map@10": 0.9661111111111111, + "mrr@1": 0.94, + "mrr@5": 0.965, + "mrr@10": 0.9661111111111111, "num_pred": 100, "num_data": 100 }, "ViDoRe_syntheticDocQA_healthcare_industry": { - "hit@1": 0.97, + "hit@1": 0.96, "hit@5": 1.0, "hit@10": 1.0, - "ndcg_linear@1": 0.97, - "ndcg_linear@5": 0.9869253606521631, - "ndcg_linear@10": 0.9869253606521631, - "ndcg_exponential@1": 0.97, - "ndcg_exponential@5": 0.9869253606521631, - "ndcg_exponential@10": 0.9869253606521631, - "precision@1": 0.97, + "ndcg_linear@1": 0.96, + "ndcg_linear@5": 0.9852371901428583, + "ndcg_linear@10": 0.9852371901428583, + "ndcg_exponential@1": 0.96, + "ndcg_exponential@5": 0.9852371901428583, + "ndcg_exponential@10": 0.9852371901428583, + "precision@1": 0.96, "precision@5": 0.19999999999999996, "precision@10": 0.09999999999999998, - "recall@1": 0.97, + "recall@1": 0.96, "recall@5": 1.0, "recall@10": 1.0, - "f1@1": 0.97, + "f1@1": 0.96, "f1@5": 0.3333333333333335, "f1@10": 0.1818181818181819, - "map@1": 0.97, - "map@5": 0.9825, - "map@10": 0.9825, - "mrr@1": 0.97, - "mrr@5": 0.9825, - "mrr@10": 0.9825, + "map@1": 0.96, + "map@5": 0.98, + "map@10": 0.98, + "mrr@1": 0.96, + "mrr@5": 0.98, + "mrr@10": 0.98, "num_pred": 100, "num_data": 100 }, "ViDoRe_esg_reports_human_labeled_v2": { "hit@1": 0.6153846153846154, - "hit@5": 0.7884615384615384, + "hit@5": 0.8076923076923077, "hit@10": 0.8653846153846154, "ndcg_linear@1": 0.5897435897435896, - "ndcg_linear@5": 0.6094691508995544, - "ndcg_linear@10": 0.635637750038373, + "ndcg_linear@5": 0.6029417470734441, + "ndcg_linear@10": 0.6330708031852817, "ndcg_exponential@1": 0.5824175824175823, - "ndcg_exponential@5": 0.6103242475099454, - "ndcg_exponential@10": 0.6356245865615132, + "ndcg_exponential@5": 0.6037852478421153, + "ndcg_exponential@10": 0.6330282294231682, "precision@1": 0.6153846153846154, - "precision@5": 0.23461538461538461, + "precision@5": 0.22307692307692306, "precision@10": 0.1423076923076923, "recall@1": 0.4133241758241758, "recall@5": 0.6478021978021977, "recall@10": 0.7252372627372627, "f1@1": 0.4552021696252465, - "f1@5": 0.3082283729342553, + "f1@5": 0.29778742499330735, "f1@10": 0.2158892483835923, "map@1": 0.6153846153846154, - "map@5": 0.5464262820512821, - "map@10": 0.5583285002035001, + "map@5": 0.5371955128205127, + "map@10": 0.5536392014070586, "mrr@1": 0.6153846153846154, - "mrr@5": 0.6769230769230768, - "mrr@10": 0.6880036630036629, + "mrr@5": 0.6798076923076922, + "mrr@10": 0.6883547008547009, "num_pred": 52, "num_data": 52 }, "ViDoRe_biomedical_lectures_v2_multilingual": { - "hit@1": 0.490625, - "hit@5": 0.796875, - "hit@10": 0.86875, - "ndcg_linear@1": 0.490625, - "ndcg_linear@5": 0.5397544908352085, - "ndcg_linear@10": 0.5693203555054913, - "ndcg_exponential@1": 0.490625, - "ndcg_exponential@5": 0.5397544908352085, - "ndcg_exponential@10": 0.5693203555054912, - "precision@1": 0.490625, - "precision@5": 0.251875, - "precision@10": 0.16140625000000003, - "recall@1": 0.2978885162019353, - "recall@5": 0.5770897790934556, - "recall@10": 0.6719354787369494, - "f1@1": 0.3395271915584416, - "f1@5": 0.3084593392718393, - "f1@10": 0.23304521375615125, - "map@1": 0.490625, - "map@5": 0.46664019097222215, - "map@10": 0.479486573188303, - "mrr@1": 0.490625, - "mrr@5": 0.6060416666666667, - "mrr@10": 0.6155078125, + "hit@1": 0.49375, + "hit@5": 0.8, + "hit@10": 0.871875, + "ndcg_linear@1": 0.49375, + "ndcg_linear@5": 0.5399189136742264, + "ndcg_linear@10": 0.5718313134173973, + "ndcg_exponential@1": 0.49375, + "ndcg_exponential@5": 0.5399189136742264, + "ndcg_exponential@10": 0.5718313134173973, + "precision@1": 0.49375, + "precision@5": 0.25250000000000006, + "precision@10": 0.1621875, + "recall@1": 0.2974978912019353, + "recall@5": 0.5759439457601222, + "recall@10": 0.6793083142347849, + "f1@1": 0.3398173701298701, + "f1@5": 0.3086736041736042, + "f1@10": 0.2348412878821886, + "map@1": 0.49375, + "map@5": 0.46628732638888887, + "map@10": 0.48031307870370377, + "mrr@1": 0.49375, + "mrr@5": 0.6090625000000001, + "mrr@10": 0.6185342261904762, "num_pred": 640, "num_data": 640 }, "ViDoRe_economics_reports_v2_multilingual": { - "hit@1": 0.5689655172413793, - "hit@5": 0.8491379310344828, - "hit@10": 0.8879310344827587, - "ndcg_linear@1": 0.5689655172413793, - "ndcg_linear@5": 0.5019343305838408, - "ndcg_linear@10": 0.47979676122532555, - "ndcg_exponential@1": 0.5689655172413793, - "ndcg_exponential@5": 0.5019343305838408, - "ndcg_exponential@10": 0.47979676122532555, - "precision@1": 0.5689655172413793, - "precision@5": 0.428448275862069, - "precision@10": 0.34698275862068967, - "recall@1": 0.08034274543141959, - "recall@5": 0.25237595480423564, - "recall@10": 0.3645843669588603, - "f1@1": 0.11905920072661083, - "f1@5": 0.25183497028873714, - "f1@10": 0.2856798311584853, - "map@1": 0.5689655172413793, - "map@5": 0.40593749999999995, - "map@10": 0.35002105419913815, - "mrr@1": 0.5689655172413793, - "mrr@5": 0.6777298850574713, - "mrr@10": 0.682835591133005, + "hit@1": 0.5905172413793104, + "hit@5": 0.8448275862068966, + "hit@10": 0.8922413793103449, + "ndcg_linear@1": 0.5905172413793104, + "ndcg_linear@5": 0.5080227487560515, + "ndcg_linear@10": 0.4842379801537703, + "ndcg_exponential@1": 0.5905172413793104, + "ndcg_exponential@5": 0.5080227487560515, + "ndcg_exponential@10": 0.4842379801537701, + "precision@1": 0.5905172413793104, + "precision@5": 0.43103448275862066, + "precision@10": 0.35, + "recall@1": 0.0860984241396845, + "recall@5": 0.25680881417497536, + "recall@10": 0.3636115260077792, + "f1@1": 0.12601322371511658, + "f1@5": 0.254177851492154, + "f1@10": 0.28670707716575644, + "map@1": 0.5905172413793104, + "map@5": 0.4117995689655172, + "map@10": 0.35474905259837014, + "mrr@1": 0.5905172413793104, + "mrr@5": 0.689080459770115, + "mrr@10": 0.695198754789272, "num_pred": 232, "num_data": 232 }, "ViDoRe_esg_reports_v2_multilingual": { - "hit@1": 0.4473684210526316, - "hit@5": 0.793859649122807, + "hit@1": 0.43859649122807015, + "hit@5": 0.8070175438596491, "hit@10": 0.8771929824561403, - "ndcg_linear@1": 0.4473684210526316, - "ndcg_linear@5": 0.5073413437187128, - "ndcg_linear@10": 0.5436696652694063, - "ndcg_exponential@1": 0.4473684210526316, - "ndcg_exponential@5": 0.5073413437187128, - "ndcg_exponential@10": 0.5436696652694063, - "precision@1": 0.4473684210526316, - "precision@5": 0.2429824561403509, - "precision@10": 0.1635964912280702, - "recall@1": 0.2503968253968254, - "recall@5": 0.5527621136173767, - "recall@10": 0.6586466165413534, - "f1@1": 0.29117386075643786, - "f1@5": 0.2958373843475797, - "f1@10": 0.22844515727846343, - "map@1": 0.4473684210526316, - "map@5": 0.4241118421052631, - "map@10": 0.4392249141371949, - "mrr@1": 0.4473684210526316, - "mrr@5": 0.5828216374269005, - "mrr@10": 0.5950205374547479, + "ndcg_linear@1": 0.43859649122807015, + "ndcg_linear@5": 0.5043004157032354, + "ndcg_linear@10": 0.5361140675028695, + "ndcg_exponential@1": 0.43859649122807015, + "ndcg_exponential@5": 0.5043004157032354, + "ndcg_exponential@10": 0.5361140675028695, + "precision@1": 0.43859649122807015, + "precision@5": 0.24385964912280705, + "precision@10": 0.16184210526315793, + "recall@1": 0.24452276524644945, + "recall@5": 0.5559784878863827, + "recall@10": 0.6495091896407685, + "f1@1": 0.28454651653381235, + "f1@5": 0.29735176195088653, + "f1@10": 0.22633426507469198, + "map@1": 0.43859649122807015, + "map@5": 0.4182590155945419, + "map@10": 0.4323955142485844, + "mrr@1": 0.43859649122807015, + "mrr@5": 0.5784356725146198, + "mrr@10": 0.5884589947089947, "num_pred": 228, "num_data": 228 }, @@ -1921,11 +1949,11 @@ "hit@5": 0.8713235294117647, "hit@10": 0.9068627450980392, "ndcg_linear@1": 0.7610294117647058, - "ndcg_linear@5": 0.8199677874725917, - "ndcg_linear@10": 0.8315054893421978, + "ndcg_linear@5": 0.8195082204383903, + "ndcg_linear@10": 0.8311094144522292, "ndcg_exponential@1": 0.7610294117647058, - "ndcg_exponential@5": 0.8199677874725917, - "ndcg_exponential@10": 0.8315054893421978, + "ndcg_exponential@5": 0.8195082204383903, + "ndcg_exponential@10": 0.8311094144522292, "precision@1": 0.7610294117647058, "precision@5": 0.17426470588235293, "precision@10": 0.09068627450980392, @@ -1936,263 +1964,207 @@ "f1@5": 0.2904411764705883, "f1@10": 0.1648841354723708, "map@1": 0.7610294117647058, - "map@5": 0.8028799019607843, - "map@10": 0.8076700124494243, + "map@5": 0.8023080065359478, + "map@10": 0.8071618230625583, "mrr@1": 0.7610294117647058, - "mrr@5": 0.8028799019607843, - "mrr@10": 0.8076700124494243, + "mrr@5": 0.8023080065359478, + "mrr@10": 0.8071618230625583, "num_pred": 816, "num_data": 816 }, "VisRAG_ChartQA": { "hit@1": 0.6825396825396826, - "hit@5": 0.9047619047619048, + "hit@5": 0.8888888888888888, "hit@10": 0.9206349206349206, "ndcg_linear@1": 0.6825396825396826, - "ndcg_linear@5": 0.7988138511124628, - "ndcg_linear@10": 0.8034021816889372, + "ndcg_linear@5": 0.7908773431759549, + "ndcg_linear@10": 0.8011197560874773, "ndcg_exponential@1": 0.6825396825396826, - "ndcg_exponential@5": 0.7988138511124628, - "ndcg_exponential@10": 0.8034021816889372, + "ndcg_exponential@5": 0.7908773431759549, + "ndcg_exponential@10": 0.8011197560874773, "precision@1": 0.6825396825396826, - "precision@5": 0.18095238095238086, + "precision@5": 0.1777777777777777, "precision@10": 0.09206349206349201, "recall@1": 0.6825396825396826, - "recall@5": 0.9047619047619048, + "recall@5": 0.8888888888888888, "recall@10": 0.9206349206349206, "f1@1": 0.6825396825396826, - "f1@5": 0.3015873015873015, + "f1@5": 0.2962962962962962, "f1@10": 0.16738816738816742, "map@1": 0.6825396825396826, - "map@5": 0.7640211640211642, - "map@10": 0.7656084656084656, + "map@5": 0.7587301587301587, + "map@10": 0.7629629629629631, "mrr@1": 0.6825396825396826, - "mrr@5": 0.7640211640211642, - "mrr@10": 0.7656084656084656, + "mrr@5": 0.7587301587301587, + "mrr@10": 0.7629629629629631, "num_pred": 63, "num_data": 63 }, "VisRAG_MP-DocVQA": { "hit@1": 0.7648054145516074, - "hit@5": 0.9035532994923858, - "hit@10": 0.937394247038917, + "hit@5": 0.9018612521150592, + "hit@10": 0.9390862944162437, "ndcg_linear@1": 0.7648054145516074, - "ndcg_linear@5": 0.8442477575170283, - "ndcg_linear@10": 0.8553129247371343, + "ndcg_linear@5": 0.8434147915784187, + "ndcg_linear@10": 0.8557510286964037, "ndcg_exponential@1": 0.7648054145516074, - "ndcg_exponential@5": 0.8442477575170283, - "ndcg_exponential@10": 0.8553129247371343, + "ndcg_exponential@5": 0.8434147915784187, + "ndcg_exponential@10": 0.8557510286964037, "precision@1": 0.7648054145516074, - "precision@5": 0.18071065989847712, - "precision@10": 0.0937394247038917, + "precision@5": 0.18037225042301183, + "precision@10": 0.09390862944162436, "recall@1": 0.7648054145516074, - "recall@5": 0.9035532994923858, - "recall@10": 0.937394247038917, + "recall@5": 0.9018612521150592, + "recall@10": 0.9390862944162437, "f1@1": 0.7648054145516074, - "f1@5": 0.3011844331641287, - "f1@10": 0.1704353176434395, + "f1@5": 0.3006204173716865, + "f1@10": 0.17074296262113525, "map@1": 0.7648054145516074, - "map@5": 0.8239988719684151, - "map@10": 0.8286332017296485, + "map@5": 0.8234348561759729, + "map@10": 0.8287050465984477, "mrr@1": 0.7648054145516074, - "mrr@5": 0.8239988719684151, - "mrr@10": 0.8286332017296485, + "mrr@5": 0.8234348561759729, + "mrr@10": 0.8287050465984477, "num_pred": 591, "num_data": 591 }, "VisRAG_SlideVQA": { - "hit@1": 0.9064748201438849, + "hit@1": 0.9136690647482014, "hit@5": 0.9802158273381295, "hit@10": 0.9856115107913669, - "ndcg_linear@1": 0.9064748201438849, - "ndcg_linear@5": 0.9337625442422413, - "ndcg_linear@10": 0.9378099940259211, - "ndcg_exponential@1": 0.9064748201438849, - "ndcg_exponential@5": 0.9337625442422413, - "ndcg_exponential@10": 0.9378099940259211, - "precision@1": 0.9064748201438849, + "ndcg_linear@1": 0.9136690647482014, + "ndcg_linear@5": 0.9367835458605615, + "ndcg_linear@10": 0.9408309956442413, + "ndcg_exponential@1": 0.9136690647482014, + "ndcg_exponential@5": 0.9367835458605615, + "ndcg_exponential@10": 0.9408309956442413, + "precision@1": 0.9136690647482014, "precision@5": 0.24244604316546764, "precision@10": 0.12302158273381293, - "recall@1": 0.783273381294964, + "recall@1": 0.7904676258992805, "recall@5": 0.966726618705036, "recall@10": 0.9775179856115108, - "f1@1": 0.8243405275779376, + "f1@1": 0.8315347721822541, "f1@5": 0.380695443645084, "f1@10": 0.21601809461521695, - "map@1": 0.9064748201438849, - "map@5": 0.9146432853717026, - "map@10": 0.9169378782688133, - "mrr@1": 0.9064748201438849, - "mrr@5": 0.9376199040767385, - "mrr@10": 0.938401421719767, + "map@1": 0.9136690647482014, + "map@5": 0.9189148681055155, + "map@10": 0.9212094610026264, + "mrr@1": 0.9136690647482014, + "mrr@5": 0.9413069544364508, + "mrr@10": 0.9420884720794792, "num_pred": 556, "num_data": 556 }, "VisRAG_InfoVQA": { "hit@1": 0.8426183844011143, - "hit@5": 0.9651810584958217, + "hit@5": 0.9665738161559888, "hit@10": 0.9874651810584958, "ndcg_linear@1": 0.8426183844011143, - "ndcg_linear@5": 0.9135602456956691, - "ndcg_linear@10": 0.9210168523969697, + "ndcg_linear@5": 0.914145300888711, + "ndcg_linear@10": 0.9210538352752424, "ndcg_exponential@1": 0.8426183844011143, - "ndcg_exponential@5": 0.9135602456956691, - "ndcg_exponential@10": 0.9210168523969697, + "ndcg_exponential@5": 0.914145300888711, + "ndcg_exponential@10": 0.9210538352752424, "precision@1": 0.8426183844011143, - "precision@5": 0.1930362116991643, + "precision@5": 0.1933147632311977, "precision@10": 0.09874651810584957, "recall@1": 0.8426183844011143, - "recall@5": 0.9651810584958217, + "recall@5": 0.9665738161559888, "recall@10": 0.9874651810584958, "f1@1": 0.8426183844011143, - "f1@5": 0.3217270194986073, + "f1@5": 0.32219127205199644, "f1@10": 0.17953912382881745, "map@1": 0.8426183844011143, - "map@5": 0.8958681522748375, - "map@10": 0.8990941548392801, + "map@5": 0.8961931290622099, + "map@10": 0.8991345005968961, "mrr@1": 0.8426183844011143, - "mrr@5": 0.8958681522748375, - "mrr@10": 0.8990941548392801, + "mrr@5": 0.8961931290622099, + "mrr@10": 0.8991345005968961, "num_pred": 718, "num_data": 718 }, "VisRAG_PlotQA": { "hit@1": 0.5052143684820394, - "hit@5": 0.764774044032445, - "hit@10": 0.8551564310544612, + "hit@5": 0.7636152954808807, + "hit@10": 0.8528389339513326, "ndcg_linear@1": 0.5052143684820394, - "ndcg_linear@5": 0.640903570079321, - "ndcg_linear@10": 0.6707054604251023, + "ndcg_linear@5": 0.6411837030695758, + "ndcg_linear@10": 0.6705343669787965, "ndcg_exponential@1": 0.5052143684820394, - "ndcg_exponential@5": 0.640903570079321, - "ndcg_exponential@10": 0.6707054604251023, + "ndcg_exponential@5": 0.6411837030695758, + "ndcg_exponential@10": 0.6705343669787965, "precision@1": 0.5052143684820394, - "precision@5": 0.15295480880648896, - "precision@10": 0.0855156431054461, + "precision@5": 0.15272305909617612, + "precision@10": 0.08528389339513325, "recall@1": 0.5052143684820394, - "recall@5": 0.764774044032445, - "recall@10": 0.8551564310544612, + "recall@5": 0.7636152954808807, + "recall@10": 0.8528389339513326, "f1@1": 0.5052143684820394, - "f1@5": 0.25492468134414836, - "f1@10": 0.15548298746444753, + "f1@5": 0.2545384318269603, + "f1@10": 0.1550616243547878, "map@1": 0.5052143684820394, - "map@5": 0.5998648126689843, - "map@10": 0.612502069193842, + "map@5": 0.6005793742757821, + "map@10": 0.6129858007320347, "mrr@1": 0.5052143684820394, - "mrr@5": 0.5998648126689843, - "mrr@10": 0.612502069193842, + "mrr@5": 0.6005793742757821, + "mrr@10": 0.6129858007320347, "num_pred": 863, "num_data": 863 }, - "ViDoSeek-page": { - "hit@1": 0.01838879159369527, - "hit@5": 0.4089316987740806, - "hit@10": 0.6401050788091068, - "ndcg_linear@1": 0.01838879159369527, - "ndcg_linear@5": 0.21620008992198805, - "ndcg_linear@10": 0.2907931583080436, - "ndcg_exponential@1": 0.01838879159369527, - "ndcg_exponential@5": 0.21620008992198805, - "ndcg_exponential@10": 0.2907931583080436, - "precision@1": 0.01838879159369527, - "precision@5": 0.08178633975481611, - "precision@10": 0.06401050788091067, - "recall@1": 0.01838879159369527, - "recall@5": 0.4089316987740806, - "recall@10": 0.6401050788091068, - "f1@1": 0.01838879159369527, - "f1@5": 0.13631056625802687, - "f1@10": 0.1163827416016558, - "map@1": 0.01838879159369527, - "map@5": 0.15302101576182137, - "map@10": 0.1837190392794596, - "mrr@1": 0.01838879159369527, - "mrr@5": 0.15302101576182137, - "mrr@10": 0.1837190392794596, + "ViDoSeek-page-fixed": { + "hit@1": 0.6506129597197898, + "hit@5": 0.9238178633975481, + "hit@10": 0.9719789842381786, + "ndcg_linear@1": 0.6506129597197898, + "ndcg_linear@5": 0.7981096675826844, + "ndcg_linear@10": 0.813855217940523, + "ndcg_exponential@1": 0.6506129597197898, + "ndcg_exponential@5": 0.7981096675826844, + "ndcg_exponential@10": 0.813855217940523, + "precision@1": 0.6506129597197898, + "precision@5": 0.1847635726795096, + "precision@10": 0.09719789842381785, + "recall@1": 0.6506129597197898, + "recall@5": 0.9238178633975481, + "recall@10": 0.9719789842381786, + "f1@1": 0.6506129597197898, + "f1@5": 0.3079392877991828, + "f1@10": 0.17672345167966888, + "map@1": 0.6506129597197898, + "map@5": 0.7561150029188558, + "map@10": 0.7627098796319461, + "mrr@1": 0.6506129597197898, + "mrr@5": 0.7561150029188558, + "mrr@10": 0.7627098796319461, "num_pred": 1142, "num_data": 1142 }, - "ViDoSeek-doc": { - "hit@1": 0.9964973730297724, - "hit@5": 0.9982486865148862, - "hit@10": 0.9982486865148862, - "ndcg_linear@1": 0.678342089900759, - "ndcg_linear@5": 0.8362374075645319, - "ndcg_linear@10": 0.8165935940149865, - "ndcg_exponential@1": 0.4510883162371779, - "ndcg_exponential@5": 0.7145068150665362, - "ndcg_exponential@10": 0.7410897811756266, - "precision@1": 0.9964973730297724, - "precision@5": 0.9262697022767075, - "precision@10": 0.8286339754816112, - "recall@1": 0.05876920449269009, - "recall@5": 0.2726143296172191, - "recall@10": 0.4766810037927378, - "f1@1": 0.11024247532054007, - "f1@5": 0.4126599828359686, - "f1@10": 0.5917511632990256, - "map@1": 0.9964973730297724, - "map@5": 0.9193257443082313, - "map@10": 0.8188673981787047, - "mrr@1": 0.9964973730297724, - "mrr@5": 0.997008172796264, - "mrr@10": 0.997008172796264, - "num_pred": 1142, - "num_data": 1142 - }, - "MMLongBench-page": { - "hit@1": 0.07637231503579953, - "hit@5": 0.3031026252983294, - "hit@10": 0.4033412887828162, - "ndcg_linear@1": 0.07637231503579953, - "ndcg_linear@5": 0.15820239639498082, - "ndcg_linear@10": 0.19395841349578616, - "ndcg_exponential@1": 0.07637231503579953, - "ndcg_exponential@5": 0.15820239639498082, - "ndcg_exponential@10": 0.19395841349578616, - "precision@1": 0.07637231503579953, - "precision@5": 0.07159904534606205, - "precision@10": 0.052983293556085914, - "recall@1": 0.048528241845664274, - "recall@5": 0.22104074326628023, - "recall@10": 0.32010313672008184, - "f1@1": 0.05683207007789346, - "f1@5": 0.10227087246601498, - "f1@10": 0.08682064083863344, - "map@1": 0.07637231503579953, - "map@5": 0.11957305754441794, - "map@10": 0.13619920114028108, - "mrr@1": 0.07637231503579953, - "mrr@5": 0.16328560063643593, - "mrr@10": 0.17665170284502024, - "num_pred": 838, - "num_data": 838 - }, - "MMLongBench-doc": { - "hit@1": 0.698090692124105, - "hit@5": 0.8007159904534606, - "hit@10": 0.8281622911694511, - "ndcg_linear@1": 0.49562450278440734, - "ndcg_linear@5": 0.5143162653023697, - "ndcg_linear@10": 0.49047862420126026, - "ndcg_exponential@1": 0.35100579611319466, - "ndcg_exponential@5": 0.43113924192770037, - "ndcg_exponential@10": 0.43753218599979815, - "precision@1": 0.698090692124105, - "precision@5": 0.5663484486873509, - "precision@10": 0.4947494033412887, - "recall@1": 0.022961848267359367, - "recall@5": 0.08932041189913144, - "recall@10": 0.15198617349694285, - "f1@1": 0.044005822895889966, - "f1@5": 0.1481301071634879, - "f1@10": 0.217939992851331, - "map@1": 0.698090692124105, - "map@5": 0.5353301511535402, - "map@10": 0.44876554785266, - "mrr@1": 0.698090692124105, - "mrr@5": 0.7382259347653143, - "mrr@10": 0.7417945221047847, + "MMLongBench-page-fixed": { + "hit@1": 0.4451073985680191, + "hit@5": 0.652744630071599, + "hit@10": 0.7100238663484487, + "ndcg_linear@1": 0.4451073985680191, + "ndcg_linear@5": 0.5132486119210347, + "ndcg_linear@10": 0.5383938147492952, + "ndcg_exponential@1": 0.4451073985680191, + "ndcg_exponential@5": 0.5132486119210347, + "ndcg_exponential@10": 0.5383938147492952, + "precision@1": 0.4451073985680191, + "precision@5": 0.16634844868735085, + "precision@10": 0.09701670644391408, + "recall@1": 0.35177150812592334, + "recall@5": 0.5731517786112059, + "recall@10": 0.6428656665530174, + "f1@1": 0.3794115126454029, + "f1@5": 0.24632973722893706, + "f1@10": 0.16210490517047424, + "map@1": 0.4451073985680191, + "map@5": 0.47191262264651285, + "map@10": 0.48471077837127957, + "mrr@1": 0.4451073985680191, + "mrr@5": 0.5276849642004773, + "mrr@10": 0.5353236163200363, "num_pred": 838, "num_data": 838 }