Upload NRMS (pytorch) trained on MIND-small — 3 seeds

README.md

@@ -19,10 +19,10 @@ NRMS news recommendation model trained on MIND-small using the
 
 | Seed | AUC | MRR | NDCG@5 | NDCG@10 |
 |------|-----|-----|--------|---------|
-| 123 | 0.6514 | 0.3041 | 0.3346 | 0.3996 |
-| 42 | 0.6503 | 0.3021 | 0.3326 | 0.3981 |
-| 456 * | 0.6551 | 0.3060 | 0.3369 | 0.4026 |
-| **mean ± std** | **0.6523±0.0021** | **0.3041±0.0016** | **0.3347±0.0018** | **0.4001±0.0019** |
+| 123 | 0.6499 | 0.3024 | 0.3341 | 0.3986 |
+| 42 | 0.6546 | 0.3061 | 0.3379 | 0.4031 |
+| 456 * | 0.6557 | 0.3073 | 0.3382 | 0.4035 |
+| **mean ± std** | **0.6534±0.0025** | **0.3052±0.0021** | **0.3367±0.0019** | **0.4017±0.0022** |
 
 \* Best seed (weights at repo root)
 

best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:793b4246af25be037c5d753152156abc1972706c8247134c62d9f3615f3cc74b
+size 31185653

seed_123/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f73339de61aa432e2177e5a8ea4f513cb2e5b7a25d41d6acbf5646152141564
+size 31185653

seed_123/test_results.json

@@ -1,8 +1,8 @@
 {
-  "loss": 4.902909890087249,
-  "auc": 0.651390856409277,
-  "mrr": 0.30410659096918746,
-  "ndcg@5": 0.3345759925405266,
-  "ndcg@10": 0.39958514718105775,
+  "loss": 4.899622355878045,
+  "auc": 0.6499077096637903,
+  "mrr": 0.30237477582625777,
+  "ndcg@5": 0.33412206372866454,
+  "ndcg@10": 0.39860992006739304,
   "num_impressions": 72903.0
 }

seed_123/training_run_summary.json

@@ -236,47 +236,49 @@
     },
     "initial_validation_metrics": {},
     "best_validation_summary": {
-        "epoch_number": 4.0,
-        "train_loss": 1.3360118222072583,
-        "average_metric_value": 0.5007627030000833,
-        "val_loss": 4.577544437914921,
-        "val_auc": 0.7281870848065012,
-        "val_mrr": 0.3817817234784785,
-        "val_ndcg@5": 0.41838593386746503,
-        "val_ndcg@10": 0.47469606984788854,
+        "epoch_number": 5.0,
+        "train_loss": 1.3224480016789306,
+        "average_metric_value": 0.5042549489402233,
+        "val_loss": 4.554530820314837,
+        "val_auc": 0.7311774575461522,
+        "val_mrr": 0.38428358669989027,
+        "val_ndcg@5": 0.42384847309241136,
+        "val_ndcg@10": 0.4777102784224392,
         "val_num_impressions": 7824.0,
         "timing": {
             "epoch_training_times": [
-                257.2602527141571,
-                258.0003807544708,
-                258.13529324531555,
-                257.61827874183655,
-                257.9994740486145,
-                257.9180815219879,
-                257.8408410549164,
-                258.0513799190521,
-                258.0490565299988
+                170.52551984786987,
+                170.40193510055542,
+                170.52726674079895,
+                170.7886083126068,
+                170.73736143112183,
+                170.55459928512573,
+                170.63723158836365,
+                170.7861430644989,
+                170.56233382225037,
+                170.56839871406555
             ],
             "epoch_validation_times": [
-                7.482877969741821,
-                7.5352623462677,
-                6.93285870552063,
-                7.12975001335144,
-                7.973841667175293,
-                6.967895269393921,
-                7.366846799850464,
-                6.555442810058594,
-                7.397475004196167
+                5.449059963226318,
+                5.667611837387085,
+                5.507464170455933,
+                5.512972116470337,
+                5.491974115371704,
+                5.519476413726807,
+                5.482134103775024,
+                5.5142576694488525,
+                5.708292722702026,
+                5.473921060562134
             ],
-            "total_training_time": 2386.75838971138
+            "total_training_time": 1761.5992500782013
         }
     },
     "final_test_metrics": {
-        "loss": 4.902909890087249,
-        "auc": 0.651390856409277,
-        "mrr": 0.30410659096918746,
-        "ndcg@5": 0.3345759925405266,
-        "ndcg@10": 0.39958514718105775,
+        "loss": 4.899622355878045,
+        "auc": 0.6499077096637903,
+        "mrr": 0.30237477582625777,
+        "ndcg@5": 0.33412206372866454,
+        "ndcg@10": 0.39860992006739304,
         "num_impressions": 72903.0
     }
 }

seed_42/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db0766a02877bd46b12770e7de74de969c35e1a167910cccb67ec259a2b03702
+size 31185653

seed_42/test_results.json

@@ -1,8 +1,8 @@
 {
-  "loss": 4.90704488716887,
-  "auc": 0.6502633736798069,
-  "mrr": 0.3021278607642923,
-  "ndcg@5": 0.3326071769634611,
-  "ndcg@10": 0.3981011491197833,
+  "loss": 4.942550075925959,
+  "auc": 0.6546368368794008,
+  "mrr": 0.3060654427094731,
+  "ndcg@5": 0.33787584545553123,
+  "ndcg@10": 0.4030953465983671,
   "num_impressions": 72903.0
 }

seed_42/training_run_summary.json

@@ -236,51 +236,59 @@
     },
     "initial_validation_metrics": {},
     "best_validation_summary": {
-        "epoch_number": 6.0,
-        "train_loss": 1.3115188907319253,
-        "average_metric_value": 0.5044032370354452,
-        "val_loss": 4.548184821488641,
-        "val_auc": 0.7310099887877498,
-        "val_mrr": 0.38385719474540975,
-        "val_ndcg@5": 0.4235970467917191,
-        "val_ndcg@10": 0.4791487178169021,
+        "epoch_number": 10.0,
+        "train_loss": 1.273207871465508,
+        "average_metric_value": 0.5112104808800277,
+        "val_loss": 4.527057873079249,
+        "val_auc": 0.7393426322797343,
+        "val_mrr": 0.3892933577817272,
+        "val_ndcg@5": 0.4302412008559449,
+        "val_ndcg@10": 0.48596473260270434,
         "val_num_impressions": 7824.0,
         "timing": {
             "epoch_training_times": [
-                257.9344186782837,
-                257.6759557723999,
-                257.6570737361908,
-                257.4844753742218,
-                257.5554881095886,
-                257.585813999176,
-                258.16601943969727,
-                257.6333541870117,
-                257.7767117023468,
-                257.66410303115845,
-                257.29704117774963
+                170.63399362564087,
+                170.24239301681519,
+                170.309086561203,
+                170.81041073799133,
+                170.47880387306213,
+                170.415442943573,
+                170.50919270515442,
+                170.68091201782227,
+                170.5483238697052,
+                170.63278102874756,
+                170.82863974571228,
+                170.750239610672,
+                170.46901392936707,
+                170.35236501693726,
+                170.57720494270325
             ],
             "epoch_validation_times": [
-                6.866185665130615,
-                6.339306116104126,
-                6.4490647315979,
-                7.0740966796875,
-                6.54096794128418,
-                7.421713590621948,
-                7.108036994934082,
-                6.734248399734497,
-                6.50663161277771,
-                7.467777729034424,
-                7.561642408370972
+                5.3182737827301025,
+                5.533895492553711,
+                5.35921311378479,
+                5.345299243927002,
+                5.660549163818359,
+                5.416679859161377,
+                5.432182312011719,
+                5.4819183349609375,
+                5.647303342819214,
+                5.431931495666504,
+                5.479611158370972,
+                5.470383882522583,
+                5.629648447036743,
+                5.457347631454468,
+                5.433174133300781
             ],
-            "total_training_time": 2910.8110024929047
+            "total_training_time": 2640.5629494190216
         }
     },
     "final_test_metrics": {
-        "loss": 4.90704488716887,
-        "auc": 0.6502633736798069,
-        "mrr": 0.3021278607642923,
-        "ndcg@5": 0.3326071769634611,
-        "ndcg@10": 0.3981011491197833,
+        "loss": 4.942550075925959,
+        "auc": 0.6546368368794008,
+        "mrr": 0.3060654427094731,
+        "ndcg@5": 0.33787584545553123,
+        "ndcg@10": 0.4030953465983671,
         "num_impressions": 72903.0
     }
 }

seed_456/best_model.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:793b4246af25be037c5d753152156abc1972706c8247134c62d9f3615f3cc74b
+size 31185653

seed_456/test_results.json

@@ -1,8 +1,8 @@
 {
-  "loss": 4.908889115781316,
-  "auc": 0.6551307987714882,
-  "mrr": 0.3060355492115324,
-  "ndcg@5": 0.3369321408156793,
-  "ndcg@10": 0.4026482962112655,
+  "loss": 4.904600863656334,
+  "auc": 0.6556561797524001,
+  "mrr": 0.30725266403497975,
+  "ndcg@5": 0.3382060685619407,
+  "ndcg@10": 0.4034838957378341,
   "num_impressions": 72903.0
 }

seed_456/training_run_summary.json

@@ -237,54 +237,54 @@
     "initial_validation_metrics": {},
     "best_validation_summary": {
         "epoch_number": 8.0,
-        "train_loss": 1.2898840191987677,
-        "average_metric_value": 0.5102701171090065,
-        "val_loss": 4.527755548914938,
-        "val_auc": 0.7371236196185322,
-        "val_mrr": 0.3894036037225983,
-        "val_ndcg@5": 0.4292683360191741,
-        "val_ndcg@10": 0.4852849090757213,
+        "train_loss": 1.2902591521040014,
+        "average_metric_value": 0.5108568923254101,
+        "val_loss": 4.527366265908255,
+        "val_auc": 0.7384775366529492,
+        "val_mrr": 0.3897771162612568,
+        "val_ndcg@5": 0.43001213182000775,
+        "val_ndcg@10": 0.4851607845674266,
         "val_num_impressions": 7824.0,
         "timing": {
             "epoch_training_times": [
-                257.83956575393677,
-                258.18328881263733,
-                258.01654505729675,
-                257.8664515018463,
-                257.9182119369507,
-                257.7222595214844,
-                257.910275220871,
-                257.8587996959686,
-                257.98480582237244,
-                257.97685956954956,
-                258.06838393211365,
-                258.0029225349426,
-                258.0660049915314
+                170.56755471229553,
+                170.59103989601135,
+                170.5441117286682,
+                170.66781497001648,
+                170.54219365119934,
+                170.46811938285828,
+                170.70471787452698,
+                171.04492950439453,
+                170.6264250278473,
+                170.76346349716187,
+                170.9272804260254,
+                170.77860307693481,
+                170.67607498168945
             ],
             "epoch_validation_times": [
-                6.457681655883789,
-                7.62598180770874,
-                7.037186145782471,
-                6.5134196281433105,
-                6.92226767539978,
-                7.230839252471924,
-                8.4147047996521,
-                7.745731830596924,
-                7.493134260177612,
-                6.909805536270142,
-                7.03028416633606,
-                6.726389169692993,
-                7.046547174453735
+                5.454913139343262,
+                5.704206943511963,
+                5.526165962219238,
+                5.636821746826172,
+                5.698307275772095,
+                5.52191162109375,
+                5.467671871185303,
+                5.518418312072754,
+                5.74641752243042,
+                5.498151063919067,
+                5.47381591796875,
+                5.721930742263794,
+                5.5079920291900635
             ],
-            "total_training_time": 3447.185839176178
+            "total_training_time": 2291.63307261467
         }
     },
     "final_test_metrics": {
-        "loss": 4.908889115781316,
-        "auc": 0.6551307987714882,
-        "mrr": 0.3060355492115324,
-        "ndcg@5": 0.3369321408156793,
-        "ndcg@10": 0.4026482962112655,
+        "loss": 4.904600863656334,
+        "auc": 0.6556561797524001,
+        "mrr": 0.30725266403497975,
+        "ndcg@5": 0.3382060685619407,
+        "ndcg@10": 0.4034838957378341,
         "num_impressions": 72903.0
     }
 }

test_results.json

@@ -1,8 +1,8 @@
 {
-  "loss": 4.908889115781316,
-  "auc": 0.6551307987714882,
-  "mrr": 0.3060355492115324,
-  "ndcg@5": 0.3369321408156793,
-  "ndcg@10": 0.4026482962112655,
+  "loss": 4.904600863656334,
+  "auc": 0.6556561797524001,
+  "mrr": 0.30725266403497975,
+  "ndcg@5": 0.3382060685619407,
+  "ndcg@10": 0.4034838957378341,
   "num_impressions": 72903.0
 }

training_run_summary.json

@@ -237,54 +237,54 @@
     "initial_validation_metrics": {},
     "best_validation_summary": {
         "epoch_number": 8.0,
-        "train_loss": 1.2898840191987677,
-        "average_metric_value": 0.5102701171090065,
-        "val_loss": 4.527755548914938,
-        "val_auc": 0.7371236196185322,
-        "val_mrr": 0.3894036037225983,
-        "val_ndcg@5": 0.4292683360191741,
-        "val_ndcg@10": 0.4852849090757213,
+        "train_loss": 1.2902591521040014,
+        "average_metric_value": 0.5108568923254101,
+        "val_loss": 4.527366265908255,
+        "val_auc": 0.7384775366529492,
+        "val_mrr": 0.3897771162612568,
+        "val_ndcg@5": 0.43001213182000775,
+        "val_ndcg@10": 0.4851607845674266,
         "val_num_impressions": 7824.0,
         "timing": {
             "epoch_training_times": [
-                257.83956575393677,
-                258.18328881263733,
-                258.01654505729675,
-                257.8664515018463,
-                257.9182119369507,
-                257.7222595214844,
-                257.910275220871,
-                257.8587996959686,
-                257.98480582237244,
-                257.97685956954956,
-                258.06838393211365,
-                258.0029225349426,
-                258.0660049915314
+                170.56755471229553,
+                170.59103989601135,
+                170.5441117286682,
+                170.66781497001648,
+                170.54219365119934,
+                170.46811938285828,
+                170.70471787452698,
+                171.04492950439453,
+                170.6264250278473,
+                170.76346349716187,
+                170.9272804260254,
+                170.77860307693481,
+                170.67607498168945
             ],
             "epoch_validation_times": [
-                6.457681655883789,
-                7.62598180770874,
-                7.037186145782471,
-                6.5134196281433105,
-                6.92226767539978,
-                7.230839252471924,
-                8.4147047996521,
-                7.745731830596924,
-                7.493134260177612,
-                6.909805536270142,
-                7.03028416633606,
-                6.726389169692993,
-                7.046547174453735
+                5.454913139343262,
+                5.704206943511963,
+                5.526165962219238,
+                5.636821746826172,
+                5.698307275772095,
+                5.52191162109375,
+                5.467671871185303,
+                5.518418312072754,
+                5.74641752243042,
+                5.498151063919067,
+                5.47381591796875,
+                5.721930742263794,
+                5.5079920291900635
             ],
-            "total_training_time": 3447.185839176178
+            "total_training_time": 2291.63307261467
         }
     },
     "final_test_metrics": {
-        "loss": 4.908889115781316,
-        "auc": 0.6551307987714882,
-        "mrr": 0.3060355492115324,
-        "ndcg@5": 0.3369321408156793,
-        "ndcg@10": 0.4026482962112655,
+        "loss": 4.904600863656334,
+        "auc": 0.6556561797524001,
+        "mrr": 0.30725266403497975,
+        "ndcg@5": 0.3382060685619407,
+        "ndcg@10": 0.4034838957378341,
         "num_impressions": 72903.0
     }
 }