Upload folder using huggingface_hub

loss_history.json

@@ -1,233 +1,47 @@
 [
   {
     "epoch": 1,
-    "phase": "P1",
-    "loss": 0.152055
+    "loss": 0.093453
   },
   {
     "epoch": 2,
-    "phase": "P1",
-    "loss": 0.126681
+    "loss": 0.092451,
+    "val_ssim": 0.8883
   },
   {
     "epoch": 3,
-    "phase": "P1",
-    "loss": 0.119891
+    "loss": 0.091656
   },
   {
     "epoch": 4,
-    "phase": "P1",
-    "loss": 0.114801
+    "loss": 0.090955,
+    "val_ssim": 0.8887
   },
   {
     "epoch": 5,
-    "phase": "P1",
-    "loss": 0.110611
+    "loss": 0.090331
   },
   {
     "epoch": 6,
-    "phase": "P1",
-    "loss": 0.107016
+    "loss": 0.089821,
+    "val_ssim": 0.8885
   },
   {
     "epoch": 7,
-    "phase": "P1",
-    "loss": 0.103401
+    "loss": 0.089413
   },
   {
     "epoch": 8,
-    "phase": "P1",
-    "loss": 0.100012
+    "loss": 0.08911,
+    "val_ssim": 0.8883
   },
   {
     "epoch": 9,
-    "phase": "P1",
-    "loss": 0.096366
+    "loss": 0.088879
   },
   {
     "epoch": 10,
-    "phase": "P1",
-    "loss": 0.09296
-  },
-  {
-    "epoch": 11,
-    "phase": "P1",
-    "loss": 0.089986
-  },
-  {
-    "epoch": 12,
-    "phase": "P1",
-    "loss": 0.087143
-  },
-  {
-    "epoch": 13,
-    "phase": "P1",
-    "loss": 0.08477
-  },
-  {
-    "epoch": 14,
-    "phase": "P1",
-    "loss": 0.083114
-  },
-  {
-    "epoch": 15,
-    "phase": "P1",
-    "loss": 0.082026
-  },
-  {
-    "epoch": 16,
-    "phase": "P2",
-    "loss": 0.122125
-  },
-  {
-    "epoch": 17,
-    "phase": "P2",
-    "loss": 0.118517
-  },
-  {
-    "epoch": 18,
-    "phase": "P2",
-    "loss": 0.115646
-  },
-  {
-    "epoch": 19,
-    "phase": "P2",
-    "loss": 0.170965
-  },
-  {
-    "epoch": 20,
-    "phase": "P2",
-    "loss": 0.163493,
-    "val_ssim": 0.8267
-  },
-  {
-    "epoch": 21,
-    "phase": "P2",
-    "loss": 0.159067
-  },
-  {
-    "epoch": 22,
-    "phase": "P2",
-    "loss": 0.237583
-  },
-  {
-    "epoch": 23,
-    "phase": "P2",
-    "loss": 0.229664
-  },
-  {
-    "epoch": 24,
-    "phase": "P2",
-    "loss": 0.221985
-  },
-  {
-    "epoch": 25,
-    "phase": "P2",
-    "loss": 0.215313,
-    "val_ssim": 0.8505
-  },
-  {
-    "epoch": 26,
-    "phase": "P2",
-    "loss": 0.208722
-  },
-  {
-    "epoch": 27,
-    "phase": "P2",
-    "loss": 0.203962
-  },
-  {
-    "epoch": 28,
-    "phase": "P2",
-    "loss": 0.198393
-  },
-  {
-    "epoch": 29,
-    "phase": "P2",
-    "loss": 0.194795
-  },
-  {
-    "epoch": 30,
-    "phase": "P2",
-    "loss": 0.191285,
-    "val_ssim": 0.8759
-  },
-  {
-    "epoch": 31,
-    "phase": "P2",
-    "loss": 0.187651
-  },
-  {
-    "epoch": 32,
-    "phase": "P2",
-    "loss": 0.184686
-  },
-  {
-    "epoch": 33,
-    "phase": "P2",
-    "loss": 0.180715
-  },
-  {
-    "epoch": 34,
-    "phase": "P2",
-    "loss": 0.176762
-  },
-  {
-    "epoch": 35,
-    "phase": "P2",
-    "loss": 0.172307,
-    "val_ssim": 0.8774
-  },
-  {
-    "epoch": 36,
-    "phase": "P2",
-    "loss": 0.167519
-  },
-  {
-    "epoch": 37,
-    "phase": "P2",
-    "loss": 0.162766
-  },
-  {
-    "epoch": 38,
-    "phase": "P2",
-    "loss": 0.157198
-  },
-  {
-    "epoch": 39,
-    "phase": "P2",
-    "loss": 0.152165
-  },
-  {
-    "epoch": 40,
-    "phase": "P2",
-    "loss": 0.147043,
-    "val_ssim": 0.886
-  },
-  {
-    "epoch": 41,
-    "phase": "P2",
-    "loss": 0.141957
-  },
-  {
-    "epoch": 42,
-    "phase": "P2",
-    "loss": 0.137481
-  },
-  {
-    "epoch": 43,
-    "phase": "P2",
-    "loss": 0.133861
-  },
-  {
-    "epoch": 44,
-    "phase": "P2",
-    "loss": 0.131363
-  },
-  {
-    "epoch": 45,
-    "phase": "P2",
-    "loss": 0.129965,
-    "val_ssim": 0.888
+    "loss": 0.088735,
+    "val_ssim": 0.8881
   }
 ]

model.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:e930868e7c620774f7f12cd9c2f056032024e50b17bd1405824daa5df80ecb6b
+oid sha256:a42ae34436c3bf75bb101d51446c6a4bb2f36b9bc6cfe1f06ebf301b519c35ad
 size 12361376

train.log

@@ -1,64 +1,24 @@
-[05:19:55] Device: cuda
-[05:19:55] Model parameters: 6,169,586, channels=[56, 112, 224]
-[05:19:55] Phase 1: Single-step (15 epochs)
-[05:19:59]   45108 sequences
-[05:20:50] P1 Epoch 1/15 | loss=0.15205
-[05:21:41] P1 Epoch 2/15 | loss=0.12668
-[05:22:29] P1 Epoch 3/15 | loss=0.11989
-[05:23:16] P1 Epoch 4/15 | loss=0.11480
-[05:24:08] P1 Epoch 5/15 | loss=0.11061
-[05:24:54] P1 Epoch 6/15 | loss=0.10702
-[05:25:46] P1 Epoch 7/15 | loss=0.10340
-[05:26:37] P1 Epoch 8/15 | loss=0.10001
-[05:27:23] P1 Epoch 9/15 | loss=0.09637
-[05:28:12] P1 Epoch 10/15 | loss=0.09296
-[05:29:02] P1 Epoch 11/15 | loss=0.08999
-[05:29:51] P1 Epoch 12/15 | loss=0.08714
-[05:30:40] P1 Epoch 13/15 | loss=0.08477
-[05:31:30] P1 Epoch 14/15 | loss=0.08311
-[05:32:17] P1 Epoch 15/15 | loss=0.08203
-[05:32:17] Phase 2: Graduated AR (30 epochs)
-[05:34:32] P2 Epoch 1/30 (steps=2) | loss=0.12213 lr=0.000500
-[05:36:49] P2 Epoch 2/30 (steps=2) | loss=0.11852 lr=0.000500
-[05:38:58] P2 Epoch 3/30 (steps=2) | loss=0.11565 lr=0.000500
-[05:44:14] P2 Epoch 4/30 (steps=4) | loss=0.17096 lr=0.000500
-[05:49:31] P2 Epoch 5/30 (steps=4) | loss=0.16349 lr=0.000500
-[05:50:57]   Val SSIM=0.8267 | {'pong': 0.7258, 'sonic': 0.8199, 'pole_position': 0.9343}
-[05:50:57]   New best! SSIM=0.8267
-[05:56:10] P2 Epoch 6/30 (steps=4) | loss=0.15907 lr=0.000500
-[06:10:41] P2 Epoch 7/30 (steps=8) | loss=0.23758 lr=0.000500
-[06:24:53] P2 Epoch 8/30 (steps=8) | loss=0.22966 lr=0.000500
-[06:39:05] P2 Epoch 9/30 (steps=8) | loss=0.22198 lr=0.000500
-[06:53:24] P2 Epoch 10/30 (steps=8) | loss=0.21531 lr=0.000500
-[06:54:54]   Val SSIM=0.8505 | {'pong': 0.7857, 'sonic': 0.8264, 'pole_position': 0.9393}
-[06:54:54]   New best! SSIM=0.8505
-[07:09:06] P2 Epoch 11/30 (steps=8) | loss=0.20872 lr=0.000500
-[07:23:28] P2 Epoch 12/30 (steps=8) | loss=0.20396 lr=0.000500
-[07:37:46] P2 Epoch 13/30 (steps=8) | loss=0.19839 lr=0.000500
-[07:52:00] P2 Epoch 14/30 (steps=8) | loss=0.19479 lr=0.000500
-[08:06:23] P2 Epoch 15/30 (steps=8) | loss=0.19129 lr=0.000500
-[08:07:46]   Val SSIM=0.8759 | {'pong': 0.8609, 'sonic': 0.8246, 'pole_position': 0.9423}
-[08:07:46]   New best! SSIM=0.8759
-[08:22:08] P2 Epoch 16/30 (steps=8) | loss=0.18765 lr=0.000495
-[08:36:25] P2 Epoch 17/30 (steps=8) | loss=0.18469 lr=0.000478
-[08:50:42] P2 Epoch 18/30 (steps=8) | loss=0.18071 lr=0.000452
-[09:04:59] P2 Epoch 19/30 (steps=8) | loss=0.17676 lr=0.000417
-[09:19:13] P2 Epoch 20/30 (steps=8) | loss=0.17231 lr=0.000375
-[09:20:41]   Val SSIM=0.8774 | {'pong': 0.8579, 'sonic': 0.8323, 'pole_position': 0.9419}
-[09:20:41]   New best! SSIM=0.8774
-[09:35:11] P2 Epoch 21/30 (steps=8) | loss=0.16752 lr=0.000327
-[09:49:35] P2 Epoch 22/30 (steps=8) | loss=0.16277 lr=0.000276
-[10:03:57] P2 Epoch 23/30 (steps=8) | loss=0.15720 lr=0.000224
-[10:18:08] P2 Epoch 24/30 (steps=8) | loss=0.15217 lr=0.000173
-[10:32:53] P2 Epoch 25/30 (steps=8) | loss=0.14704 lr=0.000125
-[10:34:17]   Val SSIM=0.8860 | {'pong': 0.876, 'sonic': 0.8357, 'pole_position': 0.9463}
-[10:34:17]   New best! SSIM=0.8860
-[10:49:35] P2 Epoch 26/30 (steps=8) | loss=0.14196 lr=0.000083
-[11:04:55] P2 Epoch 27/30 (steps=8) | loss=0.13748 lr=0.000048
-[11:20:12] P2 Epoch 28/30 (steps=8) | loss=0.13386 lr=0.000022
-[11:35:30] P2 Epoch 29/30 (steps=8) | loss=0.13136 lr=0.000010
-[11:49:54] P2 Epoch 30/30 (steps=8) | loss=0.12997 lr=0.000010
-[11:51:09]   Val SSIM=0.8880 | {'pong': 0.8813, 'sonic': 0.8349, 'pole_position': 0.9479}
-[11:51:09]   New best! SSIM=0.8880
-[11:51:09] Experiment dir: 12.4 MB
-[11:51:09] Training complete. Best val SSIM: 0.8880
+[12:02:26] Device: cuda
+[12:02:27] Loaded v10 weights from /home/coder/experiments/2026-04-14-080000-multiscale-flow-v10
+[12:02:27] Model parameters: 6,169,586, channels=[56, 112, 224]
+[12:02:27] Fine-tune: 10 epochs of 8-step AR with pure SSIM loss
+[12:02:31]   43855 sequences
+[12:15:01] Epoch 1/10 | loss=0.09345 lr=0.0000098
+[12:27:32] Epoch 2/10 | loss=0.09245 lr=0.0000091
+[12:29:02]   Val SSIM=0.8883 | {'pong': 0.8811, 'sonic': 0.8354, 'pole_position': 0.9485}
+[12:29:02]   New best! SSIM=0.8883
+[12:41:38] Epoch 3/10 | loss=0.09166 lr=0.0000081
+[12:54:16] Epoch 4/10 | loss=0.09095 lr=0.0000069
+[12:55:36]   Val SSIM=0.8887 | {'pong': 0.8824, 'sonic': 0.8352, 'pole_position': 0.9486}
+[12:55:36]   New best! SSIM=0.8887
+[13:08:12] Epoch 5/10 | loss=0.09033 lr=0.0000055
+[13:20:56] Epoch 6/10 | loss=0.08982 lr=0.0000041
+[13:22:21]   Val SSIM=0.8885 | {'pong': 0.8824, 'sonic': 0.8347, 'pole_position': 0.9483}
+[13:35:14] Epoch 7/10 | loss=0.08941 lr=0.0000029
+[13:48:14] Epoch 8/10 | loss=0.08911 lr=0.0000019
+[13:49:31]   Val SSIM=0.8883 | {'pong': 0.8822, 'sonic': 0.8344, 'pole_position': 0.9484}
+[14:02:17] Epoch 9/10 | loss=0.08888 lr=0.0000012
+[14:14:47] Epoch 10/10 | loss=0.08874 lr=0.0000010
+[14:16:07]   Val SSIM=0.8881 | {'pong': 0.8815, 'sonic': 0.8343, 'pole_position': 0.9485}
+[14:16:07] Experiment dir: 12.4 MB
+[14:16:07] Training complete. Best val SSIM: 0.8887