Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,288 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/run-ayvu4rxe.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/run-zu4qn8u5.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/run-ou1t8ei4.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/no/AttR2Unet/wandb_logs/wandb/run-20250630_035845-4hs9k1ow/run-4hs9k1ow.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/no/AttUnet/wandb_logs/wandb/run-20250629_234132-90fs14ae/run-90fs14ae.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/no/R2Unet/wandb_logs/wandb/run-20250629_152513-3gek0m8f/run-3gek0m8f.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/no/Unet/wandb_logs/wandb/run-20250629_094423-3gyhuc3y/run-3gyhuc3y.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/rad/AttR2Unet/wandb_logs/wandb/run-20250630_113217-6uarfofq/run-6uarfofq.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/rad/AttR2Unet/wandb_logs/wandb/run-20250702_084844-qmvwj7fn/run-qmvwj7fn.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/rad/AttR2Unet/wandb_logs/wandb/run-20250703_083558-ugh1t2xr/run-ugh1t2xr.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/rad/AttUnet/wandb_logs/wandb/run-20250630_113004-s279hm9f/run-s279hm9f.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/rad/AttUnet/wandb_logs/wandb/run-20250702_141443-m4k0y6lc/run-m4k0y6lc.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/rad/Unet/wandb_logs/wandb/run-20250630_112835-zv9w1m7y/run-zv9w1m7y.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/rad/Unet/wandb_logs/wandb/run-20250704_085333-18vo33tu/run-18vo33tu.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/sat/AttR2Unet/wandb_logs/wandb/run-20250630_132749-wr7xm3r0/run-wr7xm3r0.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/sat/AttR2Unet/wandb_logs/wandb/run-20250701_212922-wrg7j8xt/run-wrg7j8xt.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/sat/AttUnet/wandb_logs/wandb/run-20250630_132721-rs1w3759/run-rs1w3759.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/sat/AttUnet/wandb_logs/wandb/run-20250701_212853-83dwdy8x/run-83dwdy8x.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/sat/R2Unet/wandb_logs/wandb/run-20250630_132702-np4jx6ik/run-np4jx6ik.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/sat/R2Unet/wandb_logs/wandb/run-20250701_104813-jkiuw7th/run-jkiuw7th.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/sat/Unet/wandb_logs/wandb/run-20250630_131930-rh821y8d/run-rh821y8d.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/sat/Unet/wandb_logs/wandb/run-20250630_132639-26nmrtdb/run-26nmrtdb.wandb filter=lfs diff=lfs merge=lfs -text
+checkpoint/sat/Unet/wandb_logs/wandb/run-20250701_084003-9u1q7azh/run-9u1q7azh.wandb filter=lfs diff=lfs merge=lfs -text
+enwik8/TrainModelgpt2-lr0.00025-step10000000-warm0-size48-layer12-embd512-heads8/best_24000/ocdbt.process_0/d/8b3e47b25785d8d3a896199caf8dd6e4 filter=lfs diff=lfs merge=lfs -text
+enwik8/TrainModelgpt2-lr0.00025-step10000000-warm0-size48-layer12-embd512-heads8/best_24000/ocdbt.process_0/d/98bb249a2237a3705f8da239d955be13 filter=lfs diff=lfs merge=lfs -text
+enwik8/TrainModelgpt2-lr0.00025-step10000000-warm0-size48-layer12-embd512-heads8/best_24000/ocdbt.process_0/d/b4d805012eb995e465fed31b4bd18254 filter=lfs diff=lfs merge=lfs -text
+enwik8/TrainModelgpt2-lr0.00025-step10000000-warm0-size48-layer12-embd512-heads8/last_2348000/ocdbt.process_0/d/ad602c5c8cbc98b60a1312cad3ed6033 filter=lfs diff=lfs merge=lfs -text
+enwik8/TrainModelgpt2-lr0.00025-step10000000-warm0-size48-layer12-embd512-heads8/last_2348000/ocdbt.process_0/d/c600df643f03be49f133c45f2edc9c9f filter=lfs diff=lfs merge=lfs -text
+enwik8/TrainModelgpt2-lr0.00025-step400000-warm0-size48-layer12-embd512-heads8/best_24000/ocdbt.process_0/d/2a4fa35168b636d5d045e2888319b627 filter=lfs diff=lfs merge=lfs -text
+enwik8/TrainModelgpt2-lr0.00025-step400000-warm0-size48-layer12-embd512-heads8/best_24000/ocdbt.process_0/d/985139d40c29ea4aa3a9d1cc45337c01 filter=lfs diff=lfs merge=lfs -text
+enwik8/TrainModelgpt2-lr0.00025-step400000-warm0-size48-layer12-embd512-heads8/last_400000/ocdbt.process_0/d/11dfcf21aae806c457b8a512a60313d6 filter=lfs diff=lfs merge=lfs -text
+enwik8/TrainModelgpt2-lr0.00025-step400000-warm0-size48-layer12-embd512-heads8/last_400000/ocdbt.process_0/d/92b308e0e56efcbaff31b6d3a5ebc22b filter=lfs diff=lfs merge=lfs -text
+enwik8/TrainModelgpt2-lr0.00025-step400000-warm0-size48-layer12-embd512-heads8/last_400000/ocdbt.process_0/d/a6407ecb81887ce09f81bb36b58fa9e0 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step100000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_40000/ocdbt.process_0/d/025550a8fed8250bc61d3a399915223e filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step100000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_40000/ocdbt.process_0/d/73ba8a4b0d72cf5a69e8e4e1893dc2ca filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step100000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_40000/ocdbt.process_0/d/a0e4a8bb3e7c48fe279772ff964d5d1c filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step100000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_100000/ocdbt.process_0/d/1268794f05591ea8315b3623bc855d1a filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step100000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_100000/ocdbt.process_0/d/5d82fb7260fa4ea63abe062ad046b90b filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step100000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_100000/ocdbt.process_0/d/ada4105589bbbfc9b183040f2af66130 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step100000-warm2000-size96-layer12-embd512-heads8-shared1-routed0-topk0/best_8000/ocdbt.process_0/d/0df2db1de260f053aa595f540e975fcc filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step100000-warm2000-size96-layer12-embd512-heads8-shared1-routed0-topk0/best_8000/ocdbt.process_0/d/304bafc75932801fa2de9d5c42939fa0 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step100000-warm2000-size96-layer12-embd512-heads8-shared1-routed0-topk0/last_32000/ocdbt.process_0/d/72fc66c675ba796a6c3c1da4f68ea44c filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step100000-warm2000-size96-layer12-embd512-heads8-shared1-routed0-topk0/last_32000/ocdbt.process_0/d/8140d934e7a5bbd591bf7f04f46e07bd filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step50000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_48000/ocdbt.process_0/d/935902ab2417dadd4fd182710515e2fb filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step50000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_48000/ocdbt.process_0/d/ee9466e289e4386d2c65957ed569f0eb filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step50000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_48000/ocdbt.process_0/d/862e309cf32eaba25eef6fe7d8bec3d1 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step50000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_48000/ocdbt.process_0/d/f47965b593fbeb8fda0af4b5e002f49a filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step50000-warm2000-size96-layer12-embd512-heads8-shared1-routed0-topk0/best_8000/ocdbt.process_0/d/3cec0606894c225374988a4f025d9684 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step50000-warm2000-size96-layer12-embd512-heads8-shared1-routed0-topk0/best_8000/ocdbt.process_0/d/8d34640086430faea2ffe2cb7b88880d filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step50000-warm2000-size96-layer12-embd512-heads8-shared1-routed0-topk0/best_8000/ocdbt.process_0/d/aa712f07893f3c78757a307763f92099 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step50000-warm2000-size96-layer12-embd512-heads8-shared1-routed0-topk0/last_28000/ocdbt.process_0/d/3751a76b3e7a0daca13271cd1e55225d filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step50000-warm2000-size96-layer12-embd512-heads8-shared1-routed0-topk0/last_28000/ocdbt.process_0/d/c315acf97c06ffadba9ee69b34d5ddee filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step80000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_24000/ocdbt.process_0/d/4703a1a673a876b0929fa72b37490530 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step80000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_24000/ocdbt.process_0/d/bfeee07c03be74c38c6e153706ad61dd filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step80000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_68000/ocdbt.process_0/d/75e4da11cb315557a9797ef30a6014c9 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step80000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_68000/ocdbt.process_0/d/b1a525c9109eacd462024bac343284e7 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step80000-warm2000-size48-layer12-embd512-heads8-shared1-routed0-topk0/best_20000/ocdbt.process_0/d/010b5cfa0b8ada2aeaa0df150cb40d22 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step80000-warm2000-size48-layer12-embd512-heads8-shared1-routed0-topk0/best_20000/ocdbt.process_0/d/a6099b19ac7e428decb360b6bc3b2fa6 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step80000-warm2000-size48-layer12-embd512-heads8-shared1-routed0-topk0/last_28000/ocdbt.process_0/d/9796cdd606cec77356a73e309c4c1a07 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step80000-warm2000-size48-layer12-embd512-heads8-shared1-routed0-topk0/last_28000/ocdbt.process_0/d/d5619c044ef1dfce0e20bd7c89866297 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step80000-warm2000-size48-layer12-embd512-heads8-shared1-routed0-topk0/last_28000/ocdbt.process_0/d/d77600c53d5f70611635817245797658 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step90000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_24000/ocdbt.process_0/d/0154b1d74e34e223ad3aa7718b82c90b filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step90000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_24000/ocdbt.process_0/d/52080f4633df2fd76f53a2c39eebb4bb filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step90000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_24000/ocdbt.process_0/d/e5b441dba4e75a8573bf46b9917e8155 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step90000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_24000/ocdbt.process_0/d/021fc4ebde51a000195185810556fcb3 filter=lfs diff=lfs merge=lfs -text
+enwik8/lr0.00025-rope-step90000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_24000/ocdbt.process_0/d/92e8579840416e98443341c917bf8fde filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed0/best_80000/ocdbt.process_0/d/0489fd6d9d835b952c56f7565dae1240 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed0/best_80000/ocdbt.process_0/d/14853a70d935fdb0487b8bd5a493bf13 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed0/best_80000/ocdbt.process_0/d/a81ba8e75da7104fb0108cdc89bd0c6e filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed0/best_80000/ocdbt.process_0/d/c3ff8e316d8a620e41ba56f67d9e5ed0 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed0/best_80000/ocdbt.process_0/d/dcba386909cdb22933b07e7ffaecc372 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed0/last_100000/ocdbt.process_0/d/01d54713f3a9557fc28b4bf24eea2eaa filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed0/last_100000/ocdbt.process_0/d/37a8cbb217b5274591da59955abab60f filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed0/last_100000/ocdbt.process_0/d/6cbb7a1bc7a148064929fd03930b2308 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed0/last_100000/ocdbt.process_0/d/c6405ff2d5a15d68edaede82cd2fc424 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed0/last_100000/ocdbt.process_0/d/ef846b443d4d48e3d28f15e02ed88012 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed20/best_84000/ocdbt.process_0/d/a2f3deae250f7a62ec4bfb823c8be8f9 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed20/best_84000/ocdbt.process_0/d/ad9f033f9b5050d0357e64d5beccf5cf filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed20/best_84000/ocdbt.process_0/d/bee248a9c17e641e5dd327f3d748bf33 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed20/best_84000/ocdbt.process_0/d/fbd60bce318ed85db013a5bfbdb5bdc0 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed20/last_100000/ocdbt.process_0/d/80a003899ad47e90bd11b59e4cdb44bd filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed20/last_100000/ocdbt.process_0/d/8a0738d8c3c06226bad7fa141a054fbc filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed20/last_100000/ocdbt.process_0/d/9ac15fc914465428ca8c5a99ff25364a filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed20/last_100000/ocdbt.process_0/d/c28e822c84560095d93a0817ef8634f1 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed20/last_100000/ocdbt.process_0/d/ebe8f637dc26b179b91565a499229918 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed40/best_92000/ocdbt.process_0/d/62bf733b8526f2ab22d13738dd954e92 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed40/best_92000/ocdbt.process_0/d/6d46243d5b2ab0049624f8033e53a917 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed40/best_92000/ocdbt.process_0/d/6ebaf5db915c5e1cee0ab257cc6a8a0a filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed40/best_92000/ocdbt.process_0/d/7248833a3b1692aea478521ce7aa7185 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed40/best_92000/ocdbt.process_0/d/b83b30b5141237211d17b275cef554de filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed40/last_100000/ocdbt.process_0/d/88d75624d6a930a3d7a10d8e20ba8fba filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed40/last_100000/ocdbt.process_0/d/d025b5b445ab5f32d1f05eb384dac60a filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed40/last_100000/ocdbt.process_0/d/d09cfde1af6693c9b65a0099c5ea9e74 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads12-sinusoidal-seed40/last_100000/ocdbt.process_0/d/dd04da6051e487cdf3feb5bbd669dce4 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed0/best_88000/ocdbt.process_0/d/2566aa5ce9c9034368f581789c8276a0 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed0/best_88000/ocdbt.process_0/d/387bbb4d86d0589d4a8dfab8c2c079ad filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed0/best_88000/ocdbt.process_0/d/4ea9a5c7802f24ce51309c3fac179a0b filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed0/best_88000/ocdbt.process_0/d/843b58bde70a3ffb803b761de25bf152 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed0/best_88000/ocdbt.process_0/d/d2294923d978b199fdc7934879cd75b9 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed0/last_100000/ocdbt.process_0/d/0ac8065ad58a7060e365ce150b1166d7 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed0/last_100000/ocdbt.process_0/d/3bf949f725ecb5b467b53f8968d10a2a filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed0/last_100000/ocdbt.process_0/d/739a17830e3d52287b422dc3d40ec5c8 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed0/last_100000/ocdbt.process_0/d/9a8930124d2c7f16556d3c10ccea4c87 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed0/last_100000/ocdbt.process_0/d/fde799389afbb52bd43db346f2e4dccf filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed20/best_92000/ocdbt.process_0/d/387705bc70384169fc65d12da0ae983c filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed20/best_92000/ocdbt.process_0/d/56eeb99a1c7331e9f9c601aa2c4cf700 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed20/best_92000/ocdbt.process_0/d/8c499e945a9a0715c2d1b43c7e066189 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed20/best_92000/ocdbt.process_0/d/db2091b627216f80fb9283c0f744c358 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed20/last_100000/ocdbt.process_0/d/53cc911e40b19c09efebd5b1edec0566 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed20/last_100000/ocdbt.process_0/d/6e9887594d324c210f3668b094067ae8 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed20/last_100000/ocdbt.process_0/d/a17546dd7aac63fd9c2277c82009bef3 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed20/last_100000/ocdbt.process_0/d/e4cb0b77060e685066d378308858d910 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed40/best_76000/ocdbt.process_0/d/427e7f585d71e6f3473107bce881719a filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed40/best_76000/ocdbt.process_0/d/57d55eb07b9a0f04ba29d83284e088a6 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed40/best_76000/ocdbt.process_0/d/62db5b945c140cb840d71b3a22f229a9 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed40/best_76000/ocdbt.process_0/d/7e4d391eca5853999054e9f2ca56afc8 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed40/best_76000/ocdbt.process_0/d/bce7d5b87945ee8f476fcb17c1c57fff filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed40/last_100000/ocdbt.process_0/d/31bb6f889b1b2fb4ee0f5d326ca525b3 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed40/last_100000/ocdbt.process_0/d/44ed9ca11068eff2ce533ed929bb246c filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed40/last_100000/ocdbt.process_0/d/5ec485d1c5ce9a188dd00615be117204 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed40/last_100000/ocdbt.process_0/d/69721b02edc401c916fa043a0e5b2338 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads16-sinusoidal-seed40/last_100000/ocdbt.process_0/d/cdcf4e788bb8fc510f70c44d6922203e filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed0/best_96000/ocdbt.process_0/d/2773181a26c6bdf0208347fbb64c36f7 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed0/best_96000/ocdbt.process_0/d/969626b589fa4b5ea95f77e3211ab94b filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed0/best_96000/ocdbt.process_0/d/a8b0eabf6a605dc87c804a91fd131c7e filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed0/best_96000/ocdbt.process_0/d/c40051ff5f391d69f7b816c64f65053c filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed0/last_100000/ocdbt.process_0/d/51a32e52159aa988efba7e9a132ca6c1 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed0/last_100000/ocdbt.process_0/d/66fbf69b18da5e4cd36e7c67562766b3 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed0/last_100000/ocdbt.process_0/d/98834195958a6e8aab4a55deec932b5b filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed0/last_100000/ocdbt.process_0/d/cce76412b43ff2bbd6b55d40b89c8a0c filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed20/best_96000/ocdbt.process_0/d/401eeb791622183e2f09cdc625ad8521 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed20/best_96000/ocdbt.process_0/d/5404134b28e62b44eefd87e1f921b566 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed20/best_96000/ocdbt.process_0/d/7d91a5d0a57f0a59583f582dcd73fbe9 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed20/best_96000/ocdbt.process_0/d/8a5e027bb199da088ba4ea258cbee1e5 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed20/best_96000/ocdbt.process_0/d/90affdc536ebb6282e50d9ac9ddd485c filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed20/last_100000/ocdbt.process_0/d/59163efae1c7a9823984090081f538b6 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed20/last_100000/ocdbt.process_0/d/7dc1b4627b0e1d213f5308e3337469fc filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed20/last_100000/ocdbt.process_0/d/91d036897767ffa27cb1573bd114f597 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed20/last_100000/ocdbt.process_0/d/d49a04337bd9faa492275a6368e397e0 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed20/last_100000/ocdbt.process_0/d/f1b5ea38074828102173a6f6e09857da filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed40/best_92000/ocdbt.process_0/d/0f2423727881068ace7885d3e6fe440f filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed40/best_92000/ocdbt.process_0/d/9d7cb22a9e6d4a3739c72fac90a10939 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed40/best_92000/ocdbt.process_0/d/a35ec2fb52e5c1220c354ff4b3a6b437 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed40/best_92000/ocdbt.process_0/d/e8dc0507e50d640b01e656eac1a4f3e9 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed40/last_100000/ocdbt.process_0/d/3dce3c674dfc4e34451484cf416e2d92 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed40/last_100000/ocdbt.process_0/d/5c5c1dfc81986e94450ff8d4f15310c6 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed40/last_100000/ocdbt.process_0/d/80c05c89db85220ba7ca5d6f834c9f1f filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2/lr0.00025-idx0-heads8-sinusoidal-seed40/last_100000/ocdbt.process_0/d/d141d1d748634d69cb77a6e4ba3adc54 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/best_416000/ocdbt.process_0/d/0185b19890716c590f5d18c0571000eb filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/best_416000/ocdbt.process_0/d/200f6f786cbe0a6ee6054a46f50ec118 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/best_416000/ocdbt.process_0/d/6c21d57f25ade0c176d237973f960dec filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/best_416000/ocdbt.process_0/d/7d8cd9d36a9f9e8a898feca6142a88d2 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/best_416000/ocdbt.process_0/d/ace27ff0286cb843852c0aab088add0a filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/best_416000/ocdbt.process_0/d/fe61ae5ae95d5dcc69958b86d55b58df filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/last_500000/ocdbt.process_0/d/0e13461249a5f7e1c5ab4f3a3442a671 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/last_500000/ocdbt.process_0/d/2d7c867d9ae1ba5b1e948c336ee9d484 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/last_500000/ocdbt.process_0/d/5688c3556a6b2f7bbcd52a3f40189838 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/last_500000/ocdbt.process_0/d/95dc557abe1fc39b3f3c8dc00b1567b6 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/last_500000/ocdbt.process_0/d/b3dd1bd4ec4a7c26c9beac9b703f2697 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/last_500000/ocdbt.process_0/d/b7557ecb84bd811c8a6e4243460fbba0 filter=lfs diff=lfs merge=lfs -text
+lm1b/lmc-model/gpt2-lr0.00025-step500000-warm2000-size96-layer12-embd768-heads12/last_500000/ocdbt.process_0/d/eeed2d2c5fec994629a363d84f6dd9c1 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/best_8000/ocdbt.process_0/d/17cc798f2e7f8b0e97fc4f6c5d4865f2 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/best_8000/ocdbt.process_0/d/22c01f7c09118b4821fb6a147986e14c filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/best_8000/ocdbt.process_0/d/22e7f198f43dc2f7244c6b6e94bf15b1 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/best_8000/ocdbt.process_0/d/621711e82b029bc8d8c90dbfff336de0 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/best_8000/ocdbt.process_0/d/7f27e86d960998f1bce36980168b2670 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/best_8000/ocdbt.process_0/d/a4d2abb4e84261a4575ed66e0e1a39fd filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/last_8000/ocdbt.process_0/d/4d8ba59d2db9bda2938c96146ea6a44d filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/last_8000/ocdbt.process_0/d/4e854be3f0e456215ddfed5ffc998579 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/last_8000/ocdbt.process_0/d/a79a2b8f37a422d664c8dc0b62748cc8 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/last_8000/ocdbt.process_0/d/ba64153834fa5da4db2dbf12d6ff2179 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/last_8000/ocdbt.process_0/d/c63db3cf08cce1fd4923e43c8ae48405 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/last_8000/ocdbt.process_0/d/ca644b1070b50b87fd9f1aef3bf5c300 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size24-layer12-embd768-heads12/last_8000/ocdbt.process_0/d/e73e0124956312cbb76eda503710ff1d filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/best_280000/ocdbt.process_0/d/140835951d328a5e63dc16a467ad8679 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/best_280000/ocdbt.process_0/d/2730a39d2b41ec8d4ab6ae564f971f11 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/best_280000/ocdbt.process_0/d/3c1aad5c81b1f062fb8327bd4fe9ad4c filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/best_280000/ocdbt.process_0/d/53379bdeac540002850e3eebc720e765 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/best_280000/ocdbt.process_0/d/70cf6cb9dafe59cd5841b267e26519bb filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/best_280000/ocdbt.process_0/d/905b2a30ad54cce4dad88daa1556f6cb filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/best_280000/ocdbt.process_0/d/97dd2192bb331e24824e71ba49e3970a filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/last_416000/ocdbt.process_0/d/0bccf6601604a6b2f08277b6d71e477a filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/last_416000/ocdbt.process_0/d/13fe506de3d8417f1444b35c654b6534 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/last_416000/ocdbt.process_0/d/2562e36922e58e9555808b85770569e9 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/last_416000/ocdbt.process_0/d/39f12ce24d8427ae4127b223986e5971 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/last_416000/ocdbt.process_0/d/7723f88ca6d27c32df65226df05c5e54 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/last_416000/ocdbt.process_0/d/e91e567dd758862ed36eb9eb2d7f6a90 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-original-step500000-warm2000-size64-layer12-embd768-heads12/last_416000/ocdbt.process_0/d/ed785be73a6b8782a85eb8ef708d025d filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed0-topk0/best_32000/ocdbt.process_0/d/46e443cfa131605d7fabf836179f1f3a filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed0-topk0/best_32000/ocdbt.process_0/d/85197161bc0ae4eac03cda3a244ad195 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed0-topk0/best_32000/ocdbt.process_0/d/8ff45648687a9e3336a9720a7b76c101 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed0-topk0/best_32000/ocdbt.process_0/d/b7065fcda747216f22a1495970cedbcf filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed0-topk0/best_32000/ocdbt.process_0/d/eceeac38a423834f68e9697c31da1809 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed0-topk0/last_32000/ocdbt.process_0/d/2730db5b284ed1c9a4b9878071d0e811 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed0-topk0/last_32000/ocdbt.process_0/d/45c833f45ccf69acc9cfdf66b7b3e374 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed0-topk0/last_32000/ocdbt.process_0/d/526ef1c8da73684e08ed5c219e38e10c filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed0-topk0/last_32000/ocdbt.process_0/d/64445e6539f6a3eeb974059c2f41c771 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed0-topk0/last_32000/ocdbt.process_0/d/f7e76c3c24a11c4be62c11b4af72b9b1 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/best_4000/ocdbt.process_0/d/10b4818e186896d325eb8367f374fa68 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/best_4000/ocdbt.process_0/d/3e95367e350e6dbdf6e93f402f8f6c15 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/best_4000/ocdbt.process_0/d/42cd45637a2b17f38741d44ccae04ba5 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/best_4000/ocdbt.process_0/d/a20d6fea288eaaba20d34af3b6d8391f filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/best_4000/ocdbt.process_0/d/d504cde80cabbea7771159a5b9d34ea7 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/best_4000/ocdbt.process_0/d/df4b3173599a57b2cb50c5d19a9d853c filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/last_4000/ocdbt.process_0/d/04b9e548b57a15b6b72986cfa16a2891 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/last_4000/ocdbt.process_0/d/9a44a3b52372a6953128aa7b38abdc9a filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/last_4000/ocdbt.process_0/d/add3dc292d2b5d69442d6a99c0c7e040 filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/last_4000/ocdbt.process_0/d/b478cd116ba5afe76a9e0774743ed84d filter=lfs diff=lfs merge=lfs -text
+lm1b/lr0.00025-rope-step500000-warm2000-size96-layer12-embd768-heads12-shared1-routed4-topk4/last_4000/ocdbt.process_0/d/c286bae25ad766507717ddaa8b07cd4c filter=lfs diff=lfs merge=lfs -text
+mambaunet/res_0_climax_ldcast/wandb_logs/wandb/offline-run-20250709_212727-0deuirf4/run-0deuirf4.wandb filter=lfs diff=lfs merge=lfs -text
+mambaunet/res_0_climax_ldcast/wandb_logs/wandb/offline-run-20250709_221947-0dn6epwj/run-0dn6epwj.wandb filter=lfs diff=lfs merge=lfs -text
+mambaunet/res_0_climax_ldcast/wandb_logs/wandb/run-20250709_220448-w1hhj53z/run-w1hhj53z.wandb filter=lfs diff=lfs merge=lfs -text
+mambaunet/res_0_climax_ldcast/wandb_logs/wandb/run-20250709_220800-gr45pw2o/run-gr45pw2o.wandb filter=lfs diff=lfs merge=lfs -text
+mambaunet/swin_climax_ldcast/wandb_logs/wandb/run-20250710_212922-ba8saedk/run-ba8saedk.wandb filter=lfs diff=lfs merge=lfs -text
+mambaunet/swin_climax_ldcast/wandb_logs/wandb/run-20250711_092520-3t19koyr/run-3t19koyr.wandb filter=lfs diff=lfs merge=lfs -text
+text8/TrainModel-lr0.00025-step400000-warm0-size48-layer12-embd512-heads8/best_40000/ocdbt.process_0/d/2d386c1eb1754fb3aae898bf42c66aa3 filter=lfs diff=lfs merge=lfs -text
+text8/TrainModel-lr0.00025-step400000-warm0-size48-layer12-embd512-heads8/best_40000/ocdbt.process_0/d/7e172bed4c395493a467cf07a1bb7e54 filter=lfs diff=lfs merge=lfs -text
+text8/TrainModel-lr0.00025-step400000-warm0-size48-layer12-embd512-heads8/last_400000/ocdbt.process_0/d/2741a69df37b7412bc56198223a1c514 filter=lfs diff=lfs merge=lfs -text
+text8/TrainModel-lr0.00025-step400000-warm0-size48-layer12-embd512-heads8/last_400000/ocdbt.process_0/d/47e2a1f5d208578355a7c52fea8d79ba filter=lfs diff=lfs merge=lfs -text
+text8/TrainModelgpt2-lr0.00025-step10000000-warm0-size48-layer12-embd512-heads8/best_28000/ocdbt.process_0/d/07bb4ced5ef8f41506d90b79f9eda23b filter=lfs diff=lfs merge=lfs -text
+text8/TrainModelgpt2-lr0.00025-step10000000-warm0-size48-layer12-embd512-heads8/best_28000/ocdbt.process_0/d/49ede10116d0bd44c60cdf5de831e255 filter=lfs diff=lfs merge=lfs -text
+text8/TrainModelgpt2-lr0.00025-step10000000-warm0-size48-layer12-embd512-heads8/best_28000/ocdbt.process_0/d/c45af85c2c09e1b51086f3f8bf550fd5 filter=lfs diff=lfs merge=lfs -text
+text8/TrainModelgpt2-lr0.00025-step10000000-warm0-size48-layer12-embd512-heads8/last_1120000/ocdbt.process_0/d/4e6c9dfc831e8e790dfae5442d96d14c filter=lfs diff=lfs merge=lfs -text
+text8/TrainModelgpt2-lr0.00025-step10000000-warm0-size48-layer12-embd512-heads8/last_1120000/ocdbt.process_0/d/f3e308d56aa2497a3753fa856691bec6 filter=lfs diff=lfs merge=lfs -text
+text8/lr0.00025-learnable-step72000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_44000/ocdbt.process_0/d/2ada9fd39aca9e50d3515eacfb672718 filter=lfs diff=lfs merge=lfs -text
+text8/lr0.00025-learnable-step72000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_44000/ocdbt.process_0/d/383d25fc75fdfe4a1c85cfd3438f45c2 filter=lfs diff=lfs merge=lfs -text
+text8/lr0.00025-learnable-step72000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/best_44000/ocdbt.process_0/d/c0218d0932c47149cba6a4dbfc5ca003 filter=lfs diff=lfs merge=lfs -text
+text8/lr0.00025-learnable-step72000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_48000/ocdbt.process_0/d/573b205899bd1df7b944a4deb312ddc7 filter=lfs diff=lfs merge=lfs -text
+text8/lr0.00025-learnable-step72000-warm2000-size24-layer12-embd512-heads8-shared1-routed0-topk0/last_48000/ocdbt.process_0/d/8b9cec1acc545f4ce26b9391bfc973ff filter=lfs diff=lfs merge=lfs -text
+unet/checkpoint/full/AttR2Unet/wandb_logs/wandb/run-20250710_090052-6muq3dnu/run-6muq3dnu.wandb filter=lfs diff=lfs merge=lfs -text
+unet/checkpoint/full/AttUnet/wandb_logs/wandb/run-20250710_090011-7vgk0kbi/run-7vgk0kbi.wandb filter=lfs diff=lfs merge=lfs -text
+unet/checkpoint/full/R2Unet/wandb_logs/wandb/run-20250710_085958-0ct545p5/run-0ct545p5.wandb filter=lfs diff=lfs merge=lfs -text
+unet/checkpoint/full/Unet/wandb_logs/wandb/run-20250709_083016-eoz1mzb9/run-eoz1mzb9.wandb filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-learnable-step100000-warm2000-size96-layer12-embd192-heads3/best_4000/ocdbt.process_0/d/513bd1c3918817bbdac7b72b5f3159a2 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-learnable-step100000-warm2000-size96-layer12-embd192-heads3/best_4000/ocdbt.process_0/d/9ce826eeafe4f1782af46177f5ba5268 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-learnable-step100000-warm2000-size96-layer12-embd192-heads3/best_4000/ocdbt.process_0/d/ea45eb3b2a6bb72473e4480dd90ce3ba filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-learnable-step100000-warm2000-size96-layer12-embd192-heads3/last_4000/ocdbt.process_0/d/448a8d5377fbd2186cfcc30b2c5d9e9b filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-learnable-step100000-warm2000-size96-layer12-embd192-heads3/last_4000/ocdbt.process_0/d/6c2b3a5b7144b8432f5f6359553cb81e filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-learnable-step100000-warm2000-size96-layer12-embd192-heads3/last_4000/ocdbt.process_0/d/ccd3e2438c5b11345d564a223766c9e5 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-learnable-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/98d7da756acd369807ac8930ab7f3930 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-learnable-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/af9ecef028d589eebb63dfd5dd671439 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-learnable-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/a6495734e4ca6608818259e6556bbb93 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-learnable-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/c5420b699decd6d2960deadb6178b726 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step100000-warm2000-size96-layer12-embd192-heads3/best_20000/ocdbt.process_0/d/13a961f6f642209c0f078720ab254e4b filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step100000-warm2000-size96-layer12-embd192-heads3/best_20000/ocdbt.process_0/d/492ca3cd4d5fc983541c49d70305edd5 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step100000-warm2000-size96-layer12-embd192-heads3/best_20000/ocdbt.process_0/d/6b6683e8a5251db421c5e97597372127 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step100000-warm2000-size96-layer12-embd192-heads3/best_20000/ocdbt.process_0/d/e369e485c816a346fef6b319ef888682 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step100000-warm2000-size96-layer12-embd192-heads3/last_100000/ocdbt.process_0/d/36c1ad30838659c5b341dc8b8eeb4817 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step100000-warm2000-size96-layer12-embd192-heads3/last_100000/ocdbt.process_0/d/5ba2c2f05bc9f8548e75c31feb6d5100 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step100000-warm2000-size96-layer12-embd192-heads3/last_100000/ocdbt.process_0/d/b329880d18ee633161bb116aa0a243df filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/0214177865bdb2c9e58d7b6e2425f0c3 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/04c9da7005e3ed6834a873bc6043a943 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/3319917b29ab2c5ed4e3f8822e7f1f21 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/38f588a87c6fc60b5415e4051e152fb9 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/6d6bcd0c0745e2cb89ef02201ff4cc90 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/07930d69c6de8d0410e340def546eb3f filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/1c149e390051d8392f284a6af603f866 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/ec3068c3b032e1f396eaa4dc81dabcd8 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/f3cd156668a2169f00386fc819f25537 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-rope-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/fba3449692c80401e9fbf34a04939e5f filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step100000-warm2000-size96-layer12-embd192-heads3/best_28000/ocdbt.process_0/d/3d30af4c1212da43be95c08d20a44a9d filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step100000-warm2000-size96-layer12-embd192-heads3/best_28000/ocdbt.process_0/d/e5ceb7a24957be228ab32e05b4927126 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step100000-warm2000-size96-layer12-embd192-heads3/last_100000/ocdbt.process_0/d/17cff1e9e1dc552a3050387a1d01684e filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step100000-warm2000-size96-layer12-embd192-heads3/last_100000/ocdbt.process_0/d/658d34c6ed29e3f20b3b4d11d2262f8d filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step100000-warm2000-size96-layer12-embd192-heads3/last_100000/ocdbt.process_0/d/8e3d5961a85729abdc6556e4bfc0c1f6 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step100000-warm2000-size96-layer12-embd192-heads3/last_100000/ocdbt.process_0/d/b8590afc8acd4d90b6d1b132e7a42312 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step200000-warm2000-size96-layer12-embd192-heads3/best_24000/ocdbt.process_0/d/0965dfe48da0dc1a6e9fe7844d4e6afe filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step200000-warm2000-size96-layer12-embd192-heads3/best_24000/ocdbt.process_0/d/1ceb43863113499478ab143c88293f6e filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step200000-warm2000-size96-layer12-embd192-heads3/best_24000/ocdbt.process_0/d/8e940c169f3664c880b8836c0ddce1cc filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step200000-warm2000-size96-layer12-embd192-heads3/best_24000/ocdbt.process_0/d/9bf91698b7e01ec859fd4e4e932770e8 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step200000-warm2000-size96-layer12-embd192-heads3/best_24000/ocdbt.process_0/d/b42103f3e6de2d48cd73d99c306e450e filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step200000-warm2000-size96-layer12-embd192-heads3/best_24000/ocdbt.process_0/d/b634f81f1492d1c7bddc89d6def2250c filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step200000-warm2000-size96-layer12-embd192-heads3/last_108000/ocdbt.process_0/d/4d725076f07fe44bc4f93610a4dfbd2b filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step200000-warm2000-size96-layer12-embd192-heads3/last_108000/ocdbt.process_0/d/4e6ea26d913ba9110a3b127b7e74a004 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step200000-warm2000-size96-layer12-embd192-heads3/last_108000/ocdbt.process_0/d/5bd5da699614fb165e4561197b474163 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step200000-warm2000-size96-layer12-embd192-heads3/last_108000/ocdbt.process_0/d/72d31202e9cea677d734ad4187bb7f5a filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/1918f428e907687082d38fa12a1878c5 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/618a7d4dfa7c82cafb8ec20c9d5e8bec filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/6b19610fcb1d5f304ca408f7c6795480 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step50000-warm2000-size96-layer12-embd192-heads3/best_48000/ocdbt.process_0/d/ea5b69915b3c3c9c9b484f4d17aa57eb filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/37c987246857bde8e0e279239a212d40 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/7bb4d4c2e73f529bad6cc929a6761b0d filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/d3752b8146a3de68ed233cc97108e833 filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/eb4b90785ff3f4cca0882d3ba9471b6e filter=lfs diff=lfs merge=lfs -text
+wikitext103/lr0.00025-sinusoidal-step50000-warm2000-size96-layer12-embd192-heads3/last_48000/ocdbt.process_0/d/f9b64a7cebe5a1b34dbffc97d813a6df filter=lfs diff=lfs merge=lfs -text

checkpoint/full/R2Unet/wandb_logs/wandb/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-07-06T19:01:09.810575834+07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/logs/debug-core.log"}
+{"time":"2025-07-06T19:01:10.613276038+07:00","level":"INFO","msg":"stream: created new stream","id":"l9h3wdc8"}
+{"time":"2025-07-06T19:01:10.613321054+07:00","level":"INFO","msg":"stream: started","id":"l9h3wdc8"}
+{"time":"2025-07-06T19:01:10.613343827+07:00","level":"INFO","msg":"writer: Do: started","stream_id":"l9h3wdc8"}
+{"time":"2025-07-06T19:01:10.613344708+07:00","level":"INFO","msg":"sender: started","stream_id":"l9h3wdc8"}
+{"time":"2025-07-06T19:01:10.61341946+07:00","level":"INFO","msg":"handler: started","stream_id":"l9h3wdc8"}
+{"time":"2025-07-06T19:01:10.983932722+07:00","level":"INFO","msg":"Starting system monitor"}

checkpoint/full/R2Unet/wandb_logs/wandb/debug.log

@@ -0,0 +1,21 @@
+2025-07-06 19:01:09,516 INFO    MainThread:3882239 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1
+2025-07-06 19:01:09,516 INFO    MainThread:3882239 [wandb_setup.py:_flush():81] Configure stats pid to 3882239
+2025-07-06 19:01:09,516 INFO    MainThread:3882239 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/.config/wandb/settings
+2025-07-06 19:01:09,516 INFO    MainThread:3882239 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/weatherforecast/unet/wandb/settings
+2025-07-06 19:01:09,519 INFO    MainThread:3882239 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-07-06 19:01:09,519 INFO    MainThread:3882239 [wandb_init.py:setup_run_log_directory():703] Logging user logs to checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/logs/debug.log
+2025-07-06 19:01:09,521 INFO    MainThread:3882239 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/logs/debug-internal.log
+2025-07-06 19:01:09,521 INFO    MainThread:3882239 [wandb_init.py:init():831] calling init triggers
+2025-07-06 19:01:09,527 INFO    MainThread:3882239 [wandb_init.py:init():836] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-06 19:01:09,527 INFO    MainThread:3882239 [wandb_init.py:init():872] starting backend
+2025-07-06 19:01:09,744 INFO    MainThread:3882239 [wandb_init.py:init():875] sending inform_init request
+2025-07-06 19:01:09,769 INFO    MainThread:3882239 [wandb_init.py:init():883] backend started and connected
+2025-07-06 19:01:09,770 INFO    MainThread:3882239 [wandb_init.py:init():956] updated telemetry
+2025-07-06 19:01:09,772 INFO    MainThread:3882239 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout
+2025-07-06 19:01:10,953 INFO    MainThread:3882239 [wandb_init.py:init():1032] starting run threads in backend
+2025-07-06 19:01:11,054 INFO    MainThread:3882239 [wandb_run.py:_console_start():2453] atexit reg
+2025-07-06 19:01:11,054 INFO    MainThread:3882239 [wandb_run.py:_redirect():2301] redirect: wrap_raw
+2025-07-06 19:01:11,054 INFO    MainThread:3882239 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2025-07-06 19:01:11,064 INFO    MainThread:3882239 [wandb_run.py:_redirect():2393] Redirects installed.
+2025-07-06 19:01:11,086 INFO    MainThread:3882239 [wandb_init.py:init():1078] run started, returning control to user process

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/files/config.yaml

@@ -0,0 +1,29 @@
+_wandb:
+    value:
+        cli_version: 0.20.1
+        m:
+            - "1": trainer/global_step
+              "6":
+                - 3
+              "7": []
+        python_version: 3.10.18
+        t:
+            "1":
+                - 1
+                - 9
+                - 41
+                - 103
+            "2":
+                - 1
+                - 9
+                - 41
+                - 103
+            "3":
+                - 7
+                - 13
+                - 55
+                - 66
+            "4": 3.10.18
+            "5": 0.20.1
+            "12": 0.20.1
+            "13": linux-x86_64

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/files/output.log

@@ -0,0 +1,2 @@
+
+Detected KeyboardInterrupt, attempting graceful shutdown ...

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/files/requirements.txt

@@ -0,0 +1,84 @@
+async-timeout==5.0.1
+annotated-types==0.7.0
+certifi==2025.6.15
+typing_extensions==4.14.0
+nvidia-cuda-runtime-cu12==12.6.77
+packaging==25.0
+pydantic_core==2.33.2
+triton==3.3.1
+sympy==1.14.0
+nvidia-cufft-cu12==11.3.0.4
+charset-normalizer==3.4.2
+nvidia-cusparselt-cu12==0.6.3
+nvidia-cublas-cu12==12.6.4.1
+fsspec==2025.5.1
+pytorch-lightning==2.5.1.post0
+hf-xet==1.1.5
+attrs==25.3.0
+pydantic==2.11.6
+webencodings==0.5.1
+nvidia-cusolver-cu12==11.7.1.2
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+numpy==2.2.6
+torchsummary==1.5.1
+pip==25.1
+idna==3.10
+nvidia-cusparse-cu12==12.5.4.2
+nvidia-cuda-nvrtc-cu12==12.6.77
+setproctitle==1.3.6
+kaggle==1.7.4.5
+Jinja2==3.1.6
+MarkupSafe==3.0.2
+nvidia-nvtx-cu12==12.6.77
+climax==0.3.1
+wheel==0.45.1
+requests==2.32.4
+urllib3==2.4.0
+yarl==1.20.1
+rich==14.0.0
+six==1.17.0
+nvidia-cuda-cupti-cu12==12.6.80
+mpmath==1.3.0
+nvidia-nvjitlink-cu12==12.6.85
+bleach==6.2.0
+torchvision==0.22.1
+gitdb==4.0.12
+aiosignal==1.3.2
+psutil==7.0.0
+multidict==6.4.4
+pillow==11.2.1
+text-unidecode==1.3
+PyYAML==6.0.2
+typing-inspection==0.4.1
+lightning-utilities==0.14.3
+protobuf==6.31.1
+sentry-sdk==2.30.0
+typeshed_client==2.7.0
+smmap==5.0.2
+torchaudio==2.7.1
+tqdm==4.67.1
+wandb==0.20.1
+docstring_parser==0.16
+nvidia-cufile-cu12==1.11.1.6
+aiohttp==3.12.12
+nvidia-nccl-cu12==2.26.2
+mdurl==0.1.2
+huggingface-hub==0.33.0
+filelock==3.18.0
+frozenlist==1.7.0
+networkx==3.4.2
+importlib_resources==6.5.2
+platformdirs==4.3.8
+nvidia-cudnn-cu12==9.5.1.17
+torchmetrics==1.7.3
+torch==2.7.1
+Pygments==2.19.1
+python-slugify==8.0.4
+jsonargparse==4.40.0
+propcache==0.3.2
+GitPython==3.1.44
+markdown-it-py==3.0.0
+setuptools==78.1.1
+nvidia-curand-cu12==10.3.7.77
+click==8.2.1

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/files/wandb-metadata.json

@@ -0,0 +1,95 @@
+{
+  "os": "Linux-5.4.0-121-generic-x86_64-with-glibc2.31",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-07-06T11:43:24.207832Z",
+  "args": [
+    "--config",
+    "configs/R2Unet.yaml"
+  ],
+  "program": "/home/vinhbk1/weatherforecast/unet/src/train.py",
+  "codePath": "src/train.py",
+  "email": "weatherforecast1024hcmut@gmail.com",
+  "root": "checkpoint/full/R2Unet/wandb_logs",
+  "host": "ithndgx005",
+  "executable": "/cm/shared/miniconda3/envs/unet/bin/python",
+  "codePathLocal": "src/train.py",
+  "cpu_count": 128,
+  "cpu_count_logical": 256,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888635834368",
+      "used": "55198183424"
+    }
+  },
+  "memory": {
+    "total": "2164326477824"
+  },
+  "cpu": {
+    "count": 128,
+    "countLogical": 256
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-5e7b14fe-921c-ba36-2aeb-1a5825916bdd"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ae3cfea4-20a5-e014-8450-3d72c9d20c37"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-63b9f644-f11f-db81-3160-40e87114bdfc"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f38d21de-e2a4-9897-5d9a-c6c3fed95c6e"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-8450c54c-f86b-790b-28a4-c33c4d0ecf4b"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-a18dcce2-8b4b-0565-31b4-eb297b1cc4bb"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-46ccbd8d-c84f-32ae-07c3-675928ac6285"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f842f1aa-b36c-40c9-f7ba-095a709c8cb8"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.3"
+}

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_wandb":{"runtime":968}}

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/logs/debug-core.log

@@ -0,0 +1,13 @@
+{"time":"2025-07-06T18:43:24.063309321+07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpv49emrca/port-3820313.txt","pid":3820313,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-07-06T18:43:24.065024427+07:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3820313}
+{"time":"2025-07-06T18:43:24.065017264+07:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34255,"Zone":""}}
+{"time":"2025-07-06T18:43:24.197654491+07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:36036"}
+{"time":"2025-07-06T18:43:24.210954292+07:00","level":"INFO","msg":"handleInformInit: received","streamId":"ayvu4rxe","id":"127.0.0.1:36036"}
+{"time":"2025-07-06T18:43:25.257263423+07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"ayvu4rxe","id":"127.0.0.1:36036"}
+{"time":"2025-07-06T18:59:33.149551258+07:00","level":"INFO","msg":"handleInformTeardown: server teardown initiated","id":"127.0.0.1:36036"}
+{"time":"2025-07-06T18:59:33.162682843+07:00","level":"INFO","msg":"server is shutting down"}
+{"time":"2025-07-06T18:59:33.162668375+07:00","level":"INFO","msg":"connection: closing","id":"127.0.0.1:36036"}
+{"time":"2025-07-06T18:59:33.162822898+07:00","level":"INFO","msg":"connection: closed successfully","id":"127.0.0.1:36036"}
+{"time":"2025-07-06T18:59:34.816774941+07:00","level":"INFO","msg":"handleInformTeardown: server shutdown complete","id":"127.0.0.1:36036"}
+{"time":"2025-07-06T18:59:34.816792594+07:00","level":"INFO","msg":"connection: ManageConnectionData: connection closed","id":"127.0.0.1:36036"}
+{"time":"2025-07-06T18:59:34.816803004+07:00","level":"INFO","msg":"server is closed"}

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/logs/debug-internal.log

@@ -0,0 +1,15 @@
+{"time":"2025-07-06T18:43:24.261475228+07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/logs/debug-core.log"}
+{"time":"2025-07-06T18:43:25.248770088+07:00","level":"INFO","msg":"stream: created new stream","id":"ayvu4rxe"}
+{"time":"2025-07-06T18:43:25.248842646+07:00","level":"INFO","msg":"writer: Do: started","stream_id":"ayvu4rxe"}
+{"time":"2025-07-06T18:43:25.248847054+07:00","level":"INFO","msg":"sender: started","stream_id":"ayvu4rxe"}
+{"time":"2025-07-06T18:43:25.248842345+07:00","level":"INFO","msg":"handler: started","stream_id":"ayvu4rxe"}
+{"time":"2025-07-06T18:43:25.248937535+07:00","level":"INFO","msg":"stream: started","id":"ayvu4rxe"}
+{"time":"2025-07-06T18:43:25.695262434+07:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-06T18:59:33.162762393+07:00","level":"INFO","msg":"stream: closing","id":"ayvu4rxe"}
+{"time":"2025-07-06T18:59:33.162873233+07:00","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2025-07-06T18:59:33.171598728+07:00","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2025-07-06T18:59:34.483299821+07:00","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2025-07-06T18:59:34.80751794+07:00","level":"INFO","msg":"handler: closed","stream_id":"ayvu4rxe"}
+{"time":"2025-07-06T18:59:34.80756023+07:00","level":"INFO","msg":"writer: Close: closed","stream_id":"ayvu4rxe"}
+{"time":"2025-07-06T18:59:34.807575088+07:00","level":"INFO","msg":"sender: closed","stream_id":"ayvu4rxe"}
+{"time":"2025-07-06T18:59:34.816696002+07:00","level":"INFO","msg":"stream: closed","id":"ayvu4rxe"}

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/logs/debug.log

@@ -0,0 +1,22 @@
+2025-07-06 18:43:23,932 INFO    MainThread:3820313 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1
+2025-07-06 18:43:23,932 INFO    MainThread:3820313 [wandb_setup.py:_flush():81] Configure stats pid to 3820313
+2025-07-06 18:43:23,932 INFO    MainThread:3820313 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/.config/wandb/settings
+2025-07-06 18:43:23,932 INFO    MainThread:3820313 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/weatherforecast/unet/wandb/settings
+2025-07-06 18:43:23,969 INFO    MainThread:3820313 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-07-06 18:43:23,969 INFO    MainThread:3820313 [wandb_init.py:setup_run_log_directory():703] Logging user logs to checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/logs/debug.log
+2025-07-06 18:43:23,973 INFO    MainThread:3820313 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/logs/debug-internal.log
+2025-07-06 18:43:23,973 INFO    MainThread:3820313 [wandb_init.py:init():831] calling init triggers
+2025-07-06 18:43:23,978 INFO    MainThread:3820313 [wandb_init.py:init():836] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-06 18:43:23,978 INFO    MainThread:3820313 [wandb_init.py:init():872] starting backend
+2025-07-06 18:43:24,198 INFO    MainThread:3820313 [wandb_init.py:init():875] sending inform_init request
+2025-07-06 18:43:24,207 INFO    MainThread:3820313 [wandb_init.py:init():883] backend started and connected
+2025-07-06 18:43:24,208 INFO    MainThread:3820313 [wandb_init.py:init():956] updated telemetry
+2025-07-06 18:43:24,220 INFO    MainThread:3820313 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout
+2025-07-06 18:43:25,618 INFO    MainThread:3820313 [wandb_init.py:init():1032] starting run threads in backend
+2025-07-06 18:43:25,808 INFO    MainThread:3820313 [wandb_run.py:_console_start():2453] atexit reg
+2025-07-06 18:43:25,808 INFO    MainThread:3820313 [wandb_run.py:_redirect():2301] redirect: wrap_raw
+2025-07-06 18:43:25,820 INFO    MainThread:3820313 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2025-07-06 18:43:25,820 INFO    MainThread:3820313 [wandb_run.py:_redirect():2393] Redirects installed.
+2025-07-06 18:43:25,836 INFO    MainThread:3820313 [wandb_init.py:init():1078] run started, returning control to user process
+2025-07-06 18:59:33,141 INFO    MsgRouterThr:3820313 [mailbox.py:close():129] [no run ID] Closing mailbox, abandoning 1 handles.

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_184323-ayvu4rxe/run-ayvu4rxe.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f83a4f880a16e6d81f3f620c021b0be23cb8b0e7dd823a75a44e8b3d4f61118
+size 222191

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/files/requirements.txt

@@ -0,0 +1,84 @@
+async-timeout==5.0.1
+annotated-types==0.7.0
+certifi==2025.6.15
+typing_extensions==4.14.0
+nvidia-cuda-runtime-cu12==12.6.77
+packaging==25.0
+pydantic_core==2.33.2
+triton==3.3.1
+sympy==1.14.0
+nvidia-cufft-cu12==11.3.0.4
+charset-normalizer==3.4.2
+nvidia-cusparselt-cu12==0.6.3
+nvidia-cublas-cu12==12.6.4.1
+fsspec==2025.5.1
+pytorch-lightning==2.5.1.post0
+hf-xet==1.1.5
+attrs==25.3.0
+pydantic==2.11.6
+webencodings==0.5.1
+nvidia-cusolver-cu12==11.7.1.2
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+numpy==2.2.6
+torchsummary==1.5.1
+pip==25.1
+idna==3.10
+nvidia-cusparse-cu12==12.5.4.2
+nvidia-cuda-nvrtc-cu12==12.6.77
+setproctitle==1.3.6
+kaggle==1.7.4.5
+Jinja2==3.1.6
+MarkupSafe==3.0.2
+nvidia-nvtx-cu12==12.6.77
+climax==0.3.1
+wheel==0.45.1
+requests==2.32.4
+urllib3==2.4.0
+yarl==1.20.1
+rich==14.0.0
+six==1.17.0
+nvidia-cuda-cupti-cu12==12.6.80
+mpmath==1.3.0
+nvidia-nvjitlink-cu12==12.6.85
+bleach==6.2.0
+torchvision==0.22.1
+gitdb==4.0.12
+aiosignal==1.3.2
+psutil==7.0.0
+multidict==6.4.4
+pillow==11.2.1
+text-unidecode==1.3
+PyYAML==6.0.2
+typing-inspection==0.4.1
+lightning-utilities==0.14.3
+protobuf==6.31.1
+sentry-sdk==2.30.0
+typeshed_client==2.7.0
+smmap==5.0.2
+torchaudio==2.7.1
+tqdm==4.67.1
+wandb==0.20.1
+docstring_parser==0.16
+nvidia-cufile-cu12==1.11.1.6
+aiohttp==3.12.12
+nvidia-nccl-cu12==2.26.2
+mdurl==0.1.2
+huggingface-hub==0.33.0
+filelock==3.18.0
+frozenlist==1.7.0
+networkx==3.4.2
+importlib_resources==6.5.2
+platformdirs==4.3.8
+nvidia-cudnn-cu12==9.5.1.17
+torchmetrics==1.7.3
+torch==2.7.1
+Pygments==2.19.1
+python-slugify==8.0.4
+jsonargparse==4.40.0
+propcache==0.3.2
+GitPython==3.1.44
+markdown-it-py==3.0.0
+setuptools==78.1.1
+nvidia-curand-cu12==10.3.7.77
+click==8.2.1

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/files/wandb-metadata.json

@@ -0,0 +1,95 @@
+{
+  "os": "Linux-5.4.0-121-generic-x86_64-with-glibc2.31",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-07-06T12:01:09.769961Z",
+  "args": [
+    "--config",
+    "configs/R2Unet.yaml"
+  ],
+  "program": "/home/vinhbk1/weatherforecast/unet/src/train.py",
+  "codePath": "src/train.py",
+  "email": "weatherforecast1024hcmut@gmail.com",
+  "root": "checkpoint/full/R2Unet/wandb_logs",
+  "host": "ithndgx005",
+  "executable": "/cm/shared/miniconda3/envs/unet/bin/python",
+  "codePathLocal": "src/train.py",
+  "cpu_count": 128,
+  "cpu_count_logical": 256,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888635834368",
+      "used": "55198793728"
+    }
+  },
+  "memory": {
+    "total": "2164326477824"
+  },
+  "cpu": {
+    "count": 128,
+    "countLogical": 256
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-5e7b14fe-921c-ba36-2aeb-1a5825916bdd"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ae3cfea4-20a5-e014-8450-3d72c9d20c37"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-63b9f644-f11f-db81-3160-40e87114bdfc"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f38d21de-e2a4-9897-5d9a-c6c3fed95c6e"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-8450c54c-f86b-790b-28a4-c33c4d0ecf4b"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-a18dcce2-8b4b-0565-31b4-eb297b1cc4bb"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-46ccbd8d-c84f-32ae-07c3-675928ac6285"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f842f1aa-b36c-40c9-f7ba-095a709c8cb8"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.3"
+}

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-07-06T19:01:09.590518991+07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpua07n3g6/port-3882239.txt","pid":3882239,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-07-06T19:01:09.591742607+07:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3882239}
+{"time":"2025-07-06T19:01:09.591753127+07:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":37795,"Zone":""}}
+{"time":"2025-07-06T19:01:09.744738814+07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:39938"}
+{"time":"2025-07-06T19:01:09.771082264+07:00","level":"INFO","msg":"handleInformInit: received","streamId":"l9h3wdc8","id":"127.0.0.1:39938"}
+{"time":"2025-07-06T19:01:10.613326885+07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"l9h3wdc8","id":"127.0.0.1:39938"}
+{"time":"2025-07-06T19:05:58.406535027+07:00","level":"INFO","msg":"Parent process exited, terminating service process."}

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-07-06T19:01:09.810575834+07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/logs/debug-core.log"}
+{"time":"2025-07-06T19:01:10.613276038+07:00","level":"INFO","msg":"stream: created new stream","id":"l9h3wdc8"}
+{"time":"2025-07-06T19:01:10.613321054+07:00","level":"INFO","msg":"stream: started","id":"l9h3wdc8"}
+{"time":"2025-07-06T19:01:10.613343827+07:00","level":"INFO","msg":"writer: Do: started","stream_id":"l9h3wdc8"}
+{"time":"2025-07-06T19:01:10.613344708+07:00","level":"INFO","msg":"sender: started","stream_id":"l9h3wdc8"}
+{"time":"2025-07-06T19:01:10.61341946+07:00","level":"INFO","msg":"handler: started","stream_id":"l9h3wdc8"}
+{"time":"2025-07-06T19:01:10.983932722+07:00","level":"INFO","msg":"Starting system monitor"}

checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/logs/debug.log

@@ -0,0 +1,21 @@
+2025-07-06 19:01:09,516 INFO    MainThread:3882239 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1
+2025-07-06 19:01:09,516 INFO    MainThread:3882239 [wandb_setup.py:_flush():81] Configure stats pid to 3882239
+2025-07-06 19:01:09,516 INFO    MainThread:3882239 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/.config/wandb/settings
+2025-07-06 19:01:09,516 INFO    MainThread:3882239 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/weatherforecast/unet/wandb/settings
+2025-07-06 19:01:09,519 INFO    MainThread:3882239 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-07-06 19:01:09,519 INFO    MainThread:3882239 [wandb_init.py:setup_run_log_directory():703] Logging user logs to checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/logs/debug.log
+2025-07-06 19:01:09,521 INFO    MainThread:3882239 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to checkpoint/full/R2Unet/wandb_logs/wandb/run-20250706_190109-l9h3wdc8/logs/debug-internal.log
+2025-07-06 19:01:09,521 INFO    MainThread:3882239 [wandb_init.py:init():831] calling init triggers
+2025-07-06 19:01:09,527 INFO    MainThread:3882239 [wandb_init.py:init():836] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-06 19:01:09,527 INFO    MainThread:3882239 [wandb_init.py:init():872] starting backend
+2025-07-06 19:01:09,744 INFO    MainThread:3882239 [wandb_init.py:init():875] sending inform_init request
+2025-07-06 19:01:09,769 INFO    MainThread:3882239 [wandb_init.py:init():883] backend started and connected
+2025-07-06 19:01:09,770 INFO    MainThread:3882239 [wandb_init.py:init():956] updated telemetry
+2025-07-06 19:01:09,772 INFO    MainThread:3882239 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout
+2025-07-06 19:01:10,953 INFO    MainThread:3882239 [wandb_init.py:init():1032] starting run threads in backend
+2025-07-06 19:01:11,054 INFO    MainThread:3882239 [wandb_run.py:_console_start():2453] atexit reg
+2025-07-06 19:01:11,054 INFO    MainThread:3882239 [wandb_run.py:_redirect():2301] redirect: wrap_raw
+2025-07-06 19:01:11,054 INFO    MainThread:3882239 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2025-07-06 19:01:11,064 INFO    MainThread:3882239 [wandb_run.py:_redirect():2393] Redirects installed.
+2025-07-06 19:01:11,086 INFO    MainThread:3882239 [wandb_init.py:init():1078] run started, returning control to user process

checkpoint/full/Unet/checkpoints/epoch_004.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9712d88416e755b4b9a9f684a3f83a6c765e2d5a02ea36b7a2fad1b388830213
+size 521087

checkpoint/full/Unet/checkpoints/last.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9712d88416e755b4b9a9f684a3f83a6c765e2d5a02ea36b7a2fad1b388830213
+size 521087

checkpoint/full/Unet/csv_logs/version_0/hparams.yaml

@@ -0,0 +1,24 @@
+_instantiator: pytorch_lightning.cli.instantiate_module
+ablation: full
+batch_size: 64
+beta_1: 0.9
+beta_2: 0.99
+dir_data: /cm/archive/vinhbk1/NhaBe
+eta_min: 1.0e-08
+hours_predicted: 3
+lr: 0.0005
+max_epochs: 50
+num_workers: 8
+pin_memory: false
+pretrained_path: ''
+rad_inp_vars: precipitation
+rad_out_vars: precipitation
+rad_size: 400
+sat_inp_vars: total_precipitation
+sat_out_vars: total_precipitation
+sat_size: 25
+time_points_rad: 1
+time_points_sat: 1
+warmup_epochs: 10
+warmup_start_lr: 1.0e-08
+weight_decay: 1.0e-05

checkpoint/full/Unet/csv_logs/version_0/metrics.csv

@@ -0,0 +1,110 @@
+epoch,lr-AdamW/pg1,lr-AdamW/pg2,step,train/mse,train/rad,train/sat,val/mse,val/rad,val/sat
+,7.806511533866747e-07,7.806511533866747e-07,49,,,,,,
+0,,,49,2.260615348815918,1.7387264966964722,0.5218888521194458,,,
+,0.0004401026893701954,0.0004401026893701954,99,,,,,,
+0,,,99,2.6514506340026855,1.7208160161972046,0.9306347370147705,,,
+,0.0002303906183635172,0.0002303906183635172,149,,,,,,
+0,,,149,2.1908559799194336,1.46058988571167,0.7302661538124084,,,
+,8.764623515769576e-05,8.764623515769576e-05,199,,,,,,
+0,,,199,1.803321361541748,0.9090837836265564,0.8942375779151917,,,
+,0.0004992293488466136,0.0004992293488466136,249,,,,,,
+0,,,249,2.2010176181793213,1.4591747522354126,0.7418428659439087,,,
+,5.990731062982018e-05,5.990731062982018e-05,299,,,,,,
+0,,,299,2.134647846221924,1.461320400238037,0.6733274459838867,,,
+,0.0002696193816364834,0.0002696193816364834,349,,,,,,
+0,,,349,1.8226029872894287,1.2159926891326904,0.6066103577613831,,,
+,0.0004123637648423488,0.0004123637648423488,399,,,,,,
+0,,,399,2.458700656890869,1.6587759256362915,0.7999246120452881,,,
+,7.806511533867857e-07,7.806511533867857e-07,449,,,,,,
+0,,,449,2.0797119140625,1.3847779035568237,0.6949341297149658,,,
+0,,,489,,,,3.3349640369415283,2.2633557319641113,1.071608543395996
+,0.0004401026893701956,0.0004401026893701956,499,,,,,,
+1,,,499,1.9126521348953247,1.214236855506897,0.6984152793884277,,,
+,0.00023039061836352563,0.00023039061836352563,549,,,,,,
+1,,,549,2.7377986907958984,1.6440194845199585,1.0937793254852295,,,
+,8.764623515769884e-05,8.764623515769884e-05,599,,,,,,
+1,,,599,2.086331844329834,1.2490665912628174,0.837265133857727,,,
+,0.0004992293488466312,0.0004992293488466312,649,,,,,,
+1,,,649,2.118964672088623,1.4815689325332642,0.6373958587646484,,,
+,5.990731062982157e-05,5.990731062982157e-05,699,,,,,,
+1,,,699,2.039433240890503,1.1734329462051392,0.8660002946853638,,,
+,0.00026961938163649353,0.00026961938163649353,749,,,,,,
+1,,,749,1.6653474569320679,1.017549991607666,0.6477974653244019,,,
+,0.0004123637648423198,0.0004123637648423198,799,,,,,,
+1,,,799,1.6505846977233887,0.9599097371101379,0.690674901008606,,,
+,7.806511533868971e-07,7.806511533868971e-07,849,,,,,,
+1,,,849,1.7351620197296143,0.9010695219039917,0.8340925574302673,,,
+,0.00044010268937021277,0.00044010268937021277,899,,,,,,
+1,,,899,1.9034607410430908,1.2502593994140625,0.6532012820243835,,,
+,0.00023039061836349384,0.00023039061836349384,949,,,,,,
+1,,,949,1.8853743076324463,1.0594210624694824,0.8259531855583191,,,
+1,,,979,,,,3.1323599815368652,2.0574629306793213,1.074897050857544
+,8.764623515768775e-05,8.764623515768775e-05,999,,,,,,
+2,,,999,2.3397488594055176,1.6231653690338135,0.7165834307670593,,,
+,0.0004992293488466488,0.0004992293488466488,1049,,,,,,
+2,,,1049,1.8229830265045166,1.0955113172531128,0.727471649646759,,,
+,5.990731062981273e-05,5.990731062981273e-05,1099,,,,,,
+2,,,1099,2.0078647136688232,1.138712763786316,0.8691519498825073,,,
+,0.0002696193816364547,0.0002696193816364547,1149,,,,,,
+2,,,1149,1.549373745918274,0.9113926291465759,0.637981116771698,,,
+,0.0004123637648423333,0.0004123637648423333,1199,,,,,,
+2,,,1199,1.9529693126678467,1.1852428913116455,0.7677264213562012,,,
+,7.806511533868142e-07,7.806511533868142e-07,1249,,,,,,
+2,,,1249,2.1080148220062256,1.3082818984985352,0.7997328639030457,,,
+,0.000440102689370307,0.000440102689370307,1299,,,,,,
+2,,,1299,1.317264199256897,0.7056042551994324,0.6116599440574646,,,
+,0.00023039061836354518,0.00023039061836354518,1349,,,,,,
+2,,,1349,1.7698419094085693,1.0905606746673584,0.6792812943458557,,,
+,8.764623515770832e-05,8.764623515770832e-05,1399,,,,,,
+2,,,1399,1.3720903396606445,0.8107157349586487,0.5613746643066406,,,
+,0.0004992293488467564,0.0004992293488467564,1449,,,,,,
+2,,,1449,1.1426562070846558,0.5045422315597534,0.6381139755249023,,,
+2,,,1469,,,,3.0241756439208984,1.9639418125152588,1.0602340698242188
+,5.9907310629826644e-05,5.9907310629826644e-05,1499,,,,,,
+3,,,1499,1.5787314176559448,0.9801695942878723,0.5985618233680725,,,
+,0.0002696193816365116,0.0002696193816365116,1549,,,,,,
+3,,,1549,1.4235641956329346,0.6319566369056702,0.7916076183319092,,,
+,0.0004123637648422745,0.0004123637648422745,1599,,,,,,
+3,,,1599,1.7106356620788574,0.9816262722015381,0.7290094494819641,,,
+,7.806511533863131e-07,7.806511533863131e-07,1649,,,,,,
+3,,,1649,1.347632646560669,0.5408079028129578,0.806824803352356,,,
+,0.00044010268937007323,0.00044010268937007323,1699,,,,,,
+3,,,1699,1.4007446765899658,0.8070390224456787,0.5937055945396423,,,
+,0.0002303906183635893,0.0002303906183635893,1749,,,,,,
+3,,,1749,1.9254132509231567,0.9849134683609009,0.9404997825622559,,,
+,8.764623515772343e-05,8.764623515772343e-05,1799,,,,,,
+3,,,1799,1.814305067062378,1.1099367141723633,0.7043684124946594,,,
+,0.0004992293488466852,0.0004992293488466852,1849,,,,,,
+3,,,1849,2.2292516231536865,1.3755450248718262,0.8537065982818604,,,
+,5.990731062981895e-05,5.990731062981895e-05,1899,,,,,,
+3,,,1899,2.350019693374634,1.3968499898910522,0.9531697034835815,,,
+,0.00026961938163647467,0.00026961938163647467,1949,,,,,,
+3,,,1949,1.3172348737716675,0.6548643112182617,0.6623705625534058,,,
+3,,,1959,,,,3.008214235305786,1.955021858215332,1.053192138671875
+,0.00041236376484221106,0.00041236376484221106,1999,,,,,,
+4,,,1999,2.0337257385253906,1.0037966966629028,1.0299290418624878,,,
+,7.80651153386647e-07,7.80651153386647e-07,2049,,,,,,
+4,,,2049,1.321085810661316,0.6933162808418274,0.6277695298194885,,,
+,0.00044010268937000877,0.00044010268937000877,2099,,,,,,
+4,,,2099,1.789055347442627,1.1857273578643799,0.6033280491828918,,,
+,0.00023039061836354922,0.00023039061836354922,2149,,,,,,
+4,,,2149,2.179880142211914,1.1370419263839722,1.042838215827942,,,
+,8.764623515770644e-05,8.764623515770644e-05,2199,,,,,,
+4,,,2199,2.4413414001464844,1.4121730327606201,1.0291684865951538,,,
+,0.0004992293488469725,0.0004992293488469725,2249,,,,,,
+4,,,2249,2.15153169631958,1.2629209756851196,0.8886107206344604,,,
+,5.990731062984982e-05,5.990731062984982e-05,2299,,,,,,
+4,,,2299,1.3346858024597168,0.7142345309257507,0.6204512715339661,,,
+,0.00026961938163662873,0.00026961938163662873,2349,,,,,,
+4,,,2349,1.698438048362732,0.991798996925354,0.7066390514373779,,,
+,0.0004123637648424494,0.0004123637648424494,2399,,,,,,
+4,,,2399,1.9738185405731201,1.4765572547912598,0.4972612261772156,,,
+,7.806511533869514e-07,7.806511533869514e-07,2449,,,,,,
+4,,,2449,1.6242272853851318,0.7693199515342712,0.8549073338508606,,,
+4,,,2449,,,,3.0062015056610107,1.952803611755371,1.0533976554870605
+,0.0004401026893702612,0.0004401026893702612,2499,,,,,,
+5,,,2499,1.590540885925293,0.8464925289154053,0.7440484166145325,,,
+,0.0002303906183635175,0.0002303906183635175,2549,,,,,,
+5,,,2549,1.3881698846817017,0.9591439962387085,0.4290258586406708,,,
+,8.76462351576981e-05,8.76462351576981e-05,2599,,,,,,
+5,,,2599,1.4351027011871338,0.8542551398277283,0.5808475613594055,,,

checkpoint/full/Unet/wandb_logs/config.yaml

@@ -0,0 +1,157 @@
+# pytorch_lightning==2.5.1.post0
+seed_everything: 42
+trainer:
+  accelerator: cuda
+  strategy: ddp
+  devices:
+  - 7
+  num_nodes: 1
+  precision: 16-mixed
+  logger:
+  - class_path: pytorch_lightning.loggers.WandbLogger
+    init_args:
+      name: UnetNhaBe
+      save_dir: checkpoint/full/Unet/wandb_logs
+      version: null
+      offline: false
+      dir: null
+      id: null
+      anonymous: null
+      project: NhaBe
+      log_model: false
+      experiment: null
+      prefix: ''
+      checkpoint_name: null
+      entity: null
+      notes: null
+      tags: null
+      config: null
+      config_exclude_keys: null
+      config_include_keys: null
+      allow_val_change: null
+      group: full
+      job_type: null
+      mode: null
+      force: null
+      reinit: null
+      resume: null
+      resume_from: null
+      fork_from: null
+      save_code: null
+      tensorboard: null
+      sync_tensorboard: null
+      monitor_gym: null
+      settings: null
+  - class_path: pytorch_lightning.loggers.CSVLogger
+    init_args:
+      save_dir: checkpoint/full/Unet/csv_logs
+      name: null
+      version: null
+      prefix: ''
+      flush_logs_every_n_steps: 100
+  callbacks:
+  - class_path: pytorch_lightning.callbacks.LearningRateMonitor
+    init_args:
+      logging_interval: step
+      log_momentum: false
+      log_weight_decay: false
+  - class_path: pytorch_lightning.callbacks.ModelCheckpoint
+    init_args:
+      dirpath: checkpoint/full/Unet/checkpoints
+      filename: epoch_{epoch:03d}
+      monitor: val/mse
+      verbose: false
+      save_last: true
+      save_top_k: 1
+      save_weights_only: false
+      mode: min
+      auto_insert_metric_name: false
+      every_n_train_steps: null
+      train_time_interval: null
+      every_n_epochs: null
+      save_on_train_epoch_end: null
+      enable_version_counter: true
+  - class_path: pytorch_lightning.callbacks.EarlyStopping
+    init_args:
+      monitor: val/mse
+      min_delta: 0.0
+      patience: 10
+      verbose: false
+      mode: min
+      strict: true
+      check_finite: true
+      stopping_threshold: null
+      divergence_threshold: null
+      check_on_train_epoch_end: null
+      log_rank_zero_only: false
+  - class_path: pytorch_lightning.callbacks.RichModelSummary
+    init_args:
+      max_depth: -1
+  fast_dev_run: false
+  max_epochs: 100
+  min_epochs: 1
+  max_steps: -1
+  min_steps: null
+  max_time: null
+  limit_train_batches: null
+  limit_val_batches: null
+  limit_test_batches: null
+  limit_predict_batches: null
+  overfit_batches: 0.0
+  val_check_interval: null
+  check_val_every_n_epoch: 1
+  num_sanity_val_steps: null
+  log_every_n_steps: null
+  enable_checkpointing: true
+  enable_progress_bar: true
+  enable_model_summary: null
+  accumulate_grad_batches: 1
+  gradient_clip_val: null
+  gradient_clip_algorithm: null
+  deterministic: null
+  benchmark: null
+  inference_mode: true
+  use_distributed_sampler: true
+  profiler: null
+  detect_anomaly: false
+  barebones: false
+  plugins: null
+  sync_batchnorm: true
+  reload_dataloaders_every_n_epochs: 0
+  default_root_dir: checkpoint/full/Unet
+  model_registry: null
+model:
+  net:
+    class_path: arch.Network
+    init_args:
+      model_type: Unet
+      rad_channel: 1
+      sat_channel: 1
+      rad_size: 400
+      sat_size: 25
+  pretrained_path: ''
+  lr: 0.0005
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 1.0e-05
+  warmup_epochs: 10
+  max_epochs: 50
+  warmup_start_lr: 1.0e-08
+  eta_min: 1.0e-08
+data:
+  dir_data: /cm/archive/vinhbk1/NhaBe
+  batch_size: 64
+  hours_predicted: 3
+  num_workers: 8
+  pin_memory: false
+  time_points_rad: 1
+  time_points_sat: 1
+  sat_inp_vars: total_precipitation
+  sat_out_vars: total_precipitation
+  sat_size: 25
+  rad_inp_vars: precipitation
+  rad_out_vars: precipitation
+  rad_size: 400
+  ablation: full
+optimizer: null
+lr_scheduler: null

checkpoint/full/Unet/wandb_logs/wandb/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2025-07-07T08:52:43.786989998+07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/logs/debug-core.log"}
+{"time":"2025-07-07T08:52:44.462305768+07:00","level":"INFO","msg":"stream: created new stream","id":"ou1t8ei4"}
+{"time":"2025-07-07T08:52:44.4623492+07:00","level":"INFO","msg":"stream: started","id":"ou1t8ei4"}
+{"time":"2025-07-07T08:52:44.462346996+07:00","level":"INFO","msg":"handler: started","stream_id":"ou1t8ei4"}
+{"time":"2025-07-07T08:52:44.462369218+07:00","level":"INFO","msg":"sender: started","stream_id":"ou1t8ei4"}
+{"time":"2025-07-07T08:52:44.462364569+07:00","level":"INFO","msg":"writer: Do: started","stream_id":"ou1t8ei4"}
+{"time":"2025-07-07T08:52:45.010883933+07:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-07T08:52:45.477455488+07:00","level":"ERROR","msg":"file transfer: upload: failed to upload: 400 Bad Request","task":"DefaultUploadTask{FileKind: 1, Path: checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/files/wandb-metadata.json, Name: wandb-metadata.json, Url: https://storage.googleapis.com/wandb-production.appspot.com/weatherforecast1024/NhaBe/ou1t8ei4/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250707%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250707T015245Z&X-Goog-Expires=86399&X-Goog-Signature=048f05e8bdb2838e0fb7b05d69cc870b2a36f2196c0793d100fe1b18cf87643c4598200ef38ae8125510c15f1b4c87bc21e8ab12eb20885f8b6a93a785da00afdca79046efe1f94cb10b7dfc7a2fdb63d214f4ab8a06be67c0f01612f11fdf4181055615061117a5355978b64a3d94ef0c8cb67009554acf84e80f99906cd78e0a9a02476e4f1078a2d525ce72d3da48c6673f0fb6e7a0f03db4f3c74140aad9db78dd711dfc2b6af315e2f44a9ed2ee7b36fbd593a730acb26aa9ccdb1d3c0c9d0f9fc37eb8f0e1abc3ce1a2e1d4929abb71f26e3884cb776f8f16fefa9d01bd0292bf6fefca867079a1db1fff53f4be9828072c2b5c55ef0b2631739a2df18&X-Goog-SignedHeaders=host&X-User=weatherforecast1024hcmut, Size: 502}"}

checkpoint/full/Unet/wandb_logs/wandb/debug.log

@@ -0,0 +1,21 @@
+2025-07-07 08:52:43,118 INFO    MainThread:2461292 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1
+2025-07-07 08:52:43,119 INFO    MainThread:2461292 [wandb_setup.py:_flush():81] Configure stats pid to 2461292
+2025-07-07 08:52:43,119 INFO    MainThread:2461292 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/.config/wandb/settings
+2025-07-07 08:52:43,119 INFO    MainThread:2461292 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/weatherforecast/unet/wandb/settings
+2025-07-07 08:52:43,159 INFO    MainThread:2461292 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-07-07 08:52:43,160 INFO    MainThread:2461292 [wandb_init.py:setup_run_log_directory():703] Logging user logs to checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/logs/debug.log
+2025-07-07 08:52:43,160 INFO    MainThread:2461292 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/logs/debug-internal.log
+2025-07-07 08:52:43,169 INFO    MainThread:2461292 [wandb_init.py:init():831] calling init triggers
+2025-07-07 08:52:43,170 INFO    MainThread:2461292 [wandb_init.py:init():836] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-07 08:52:43,173 INFO    MainThread:2461292 [wandb_init.py:init():872] starting backend
+2025-07-07 08:52:43,592 INFO    MainThread:2461292 [wandb_init.py:init():875] sending inform_init request
+2025-07-07 08:52:43,627 INFO    MainThread:2461292 [wandb_init.py:init():883] backend started and connected
+2025-07-07 08:52:43,629 INFO    MainThread:2461292 [wandb_init.py:init():956] updated telemetry
+2025-07-07 08:52:43,629 INFO    MainThread:2461292 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout
+2025-07-07 08:52:44,918 INFO    MainThread:2461292 [wandb_init.py:init():1032] starting run threads in backend
+2025-07-07 08:52:45,138 INFO    MainThread:2461292 [wandb_run.py:_console_start():2453] atexit reg
+2025-07-07 08:52:45,138 INFO    MainThread:2461292 [wandb_run.py:_redirect():2301] redirect: wrap_raw
+2025-07-07 08:52:45,144 INFO    MainThread:2461292 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2025-07-07 08:52:45,151 INFO    MainThread:2461292 [wandb_run.py:_redirect():2393] Redirects installed.
+2025-07-07 08:52:45,166 INFO    MainThread:2461292 [wandb_init.py:init():1078] run started, returning control to user process

checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/files/output.log

@@ -0,0 +1,151 @@
+Number of train samples: 31336
+Number of test samples: 8092
+Number of val samples: 1409
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+┏━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓
+┃     ┃ Name                            ┃ Type          ┃ Params ┃ Mode  ┃
+┡━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩
+│ 0   │ net                             │ Network       │ 30.0 K │ train │
+│ 1   │ net.net                         │ Unet          │ 30.0 K │ train │
+│ 2   │ net.net.encoder_blocks          │ ModuleList    │  4.8 K │ train │
+│ 3   │ net.net.encoder_blocks.0        │ ConvBlock     │     66 │ train │
+│ 4   │ net.net.encoder_blocks.0.conv   │ Sequential    │     66 │ train │
+│ 5   │ net.net.encoder_blocks.0.conv.0 │ Conv2d        │     20 │ train │
+│ 6   │ net.net.encoder_blocks.0.conv.1 │ SyncBatchNorm │      4 │ train │
+│ 7   │ net.net.encoder_blocks.0.conv.2 │ ReLU          │      0 │ train │
+│ 8   │ net.net.encoder_blocks.0.conv.3 │ Conv2d        │     38 │ train │
+│ 9   │ net.net.encoder_blocks.0.conv.4 │ SyncBatchNorm │      4 │ train │
+│ 10  │ net.net.encoder_blocks.0.conv.5 │ ReLU          │      0 │ train │
+│ 11  │ net.net.encoder_blocks.1        │ ConvBlock     │    240 │ train │
+│ 12  │ net.net.encoder_blocks.1.conv   │ Sequential    │    240 │ train │
+│ 13  │ net.net.encoder_blocks.1.conv.0 │ Conv2d        │     76 │ train │
+│ 14  │ net.net.encoder_blocks.1.conv.1 │ SyncBatchNorm │      8 │ train │
+│ 15  │ net.net.encoder_blocks.1.conv.2 │ ReLU          │      0 │ train │
+│ 16  │ net.net.encoder_blocks.1.conv.3 │ Conv2d        │    148 │ train │
+│ 17  │ net.net.encoder_blocks.1.conv.4 │ SyncBatchNorm │      8 │ train │
+│ 18  │ net.net.encoder_blocks.1.conv.5 │ ReLU          │      0 │ train │
+│ 19  │ net.net.encoder_blocks.2        │ ConvBlock     │    912 │ train │
+│ 20  │ net.net.encoder_blocks.2.conv   │ Sequential    │    912 │ train │
+│ 21  │ net.net.encoder_blocks.2.conv.0 │ Conv2d        │    296 │ train │
+│ 22  │ net.net.encoder_blocks.2.conv.1 │ SyncBatchNorm │     16 │ train │
+│ 23  │ net.net.encoder_blocks.2.conv.2 │ ReLU          │      0 │ train │
+│ 24  │ net.net.encoder_blocks.2.conv.3 │ Conv2d        │    584 │ train │
+│ 25  │ net.net.encoder_blocks.2.conv.4 │ SyncBatchNorm │     16 │ train │
+│ 26  │ net.net.encoder_blocks.2.conv.5 │ ReLU          │      0 │ train │
+│ 27  │ net.net.encoder_blocks.3        │ ConvBlock     │  3.6 K │ train │
+│ 28  │ net.net.encoder_blocks.3.conv   │ Sequential    │  3.6 K │ train │
+│ 29  │ net.net.encoder_blocks.3.conv.0 │ Conv2d        │  1.2 K │ train │
+│ 30  │ net.net.encoder_blocks.3.conv.1 │ SyncBatchNorm │     32 │ train │
+│ 31  │ net.net.encoder_blocks.3.conv.2 │ ReLU          │      0 │ train │
+│ 32  │ net.net.encoder_blocks.3.conv.3 │ Conv2d        │  2.3 K │ train │
+│ 33  │ net.net.encoder_blocks.3.conv.4 │ SyncBatchNorm │     32 │ train │
+│ 34  │ net.net.encoder_blocks.3.conv.5 │ ReLU          │      0 │ train │
+│ 35  │ net.net.pools                   │ ModuleList    │      0 │ train │
+│ 36  │ net.net.pools.0                 │ MaxPool2d     │      0 │ train │
+│ 37  │ net.net.pools.1                 │ MaxPool2d     │      0 │ train ���
+│ 38  │ net.net.pools.2                 │ MaxPool2d     │      0 │ train │
+│ 39  │ net.net.pools.3                 │ MaxPool2d     │      0 │ train │
+│ 40  │ net.net.mid_conv_1              │ single_conv   │  2.4 K │ train │
+│ 41  │ net.net.mid_conv_1.conv         │ Sequential    │  2.4 K │ train │
+│ 42  │ net.net.mid_conv_1.conv.0       │ Conv2d        │  2.3 K │ train │
+│ 43  │ net.net.mid_conv_1.conv.1       │ SyncBatchNorm │     32 │ train │
+│ 44  │ net.net.mid_conv_1.conv.2       │ ReLU          │      0 │ train │
+│ 45  │ net.net.mid_conv_2              │ single_conv   │    192 │ train │
+│ 46  │ net.net.mid_conv_2.conv         │ Sequential    │    192 │ train │
+│ 47  │ net.net.mid_conv_2.conv.0       │ Conv2d        │    160 │ train │
+│ 48  │ net.net.mid_conv_2.conv.1       │ SyncBatchNorm │     32 │ train │
+│ 49  │ net.net.mid_conv_2.conv.2       │ ReLU          │      0 │ train │
+│ 50  │ net.net.mid_merge               │ ConvBlock     │  7.0 K │ train │
+│ 51  │ net.net.mid_merge.conv          │ Sequential    │  7.0 K │ train │
+│ 52  │ net.net.mid_merge.conv.0        │ Conv2d        │  4.6 K │ train │
+│ 53  │ net.net.mid_merge.conv.1        │ SyncBatchNorm │     32 │ train │
+│ 54  │ net.net.mid_merge.conv.2        │ ReLU          │      0 │ train │
+│ 55  │ net.net.mid_merge.conv.3        │ Conv2d        │  2.3 K │ train │
+│ 56  │ net.net.mid_merge.conv.4        │ SyncBatchNorm │     32 │ train │
+│ 57  │ net.net.mid_merge.conv.5        │ ReLU          │      0 │ train │
+│ 58  │ net.net.up_convs                │ ModuleList    │  6.2 K │ train │
+│ 59  │ net.net.up_convs.0              │ UpConv        │  4.7 K │ train │
+│ 60  │ net.net.up_convs.0.up           │ Sequential    │  4.7 K │ train │
+│ 61  │ net.net.up_convs.0.up.0         │ Upsample      │      0 │ train │
+│ 62  │ net.net.up_convs.0.up.1         │ Conv2d        │  4.6 K │ train │
+│ 63  │ net.net.up_convs.0.up.2         │ SyncBatchNorm │     32 │ train │
+│ 64  │ net.net.up_convs.0.up.3         │ ReLU          │      0 │ train │
+│ 65  │ net.net.up_convs.1              │ UpConv        │  1.2 K │ train │
+│ 66  │ net.net.up_convs.1.up           │ Sequential    │  1.2 K │ train │
+│ 67  │ net.net.up_convs.1.up.0         │ Upsample      │      0 │ train │
+│ 68  │ net.net.up_convs.1.up.1         │ Conv2d        │  1.2 K │ train │
+│ 69  │ net.net.up_convs.1.up.2         │ SyncBatchNorm │     16 │ train │
+│ 70  │ net.net.up_convs.1.up.3         │ ReLU          │      0 │ train │
+│ 71  │ net.net.up_convs.2              │ UpConv        │    300 │ train │
+│ 72  │ net.net.up_convs.2.up           │ Sequential    │    300 │ train │
+│ 73  │ net.net.up_convs.2.up.0         │ Upsample      │      0 │ train │
+│ 74  │ net.net.up_convs.2.up.1         │ Conv2d        │    292 │ train │
+│ 75  │ net.net.up_convs.2.up.2         │ SyncBatchNorm │      8 │ train │
+│ 76  │ net.net.up_convs.2.up.3         │ ReLU          │      0 │ train │
+│ 77  │ net.net.up_convs.3              │ UpConv        │     78 │ train │
+│ 78  │ net.net.up_convs.3.up           │ Sequential    │     78 │ train │
+│ 79  │ net.net.up_convs.3.up.0         │ Upsample      │      0 │ train │
+│ 80  │ net.net.up_convs.3.up.1         │ Conv2d        │     74 │ train │
+│ 81  │ net.net.up_convs.3.up.2         │ SyncBatchNorm │      4 │ train │
+│ 82  │ net.net.up_convs.3.up.3         │ ReLU          │      0 │ train │
+│ 83  │ net.net.decoder_blocks          │ ModuleList    │  9.4 K │ train │
+│ 84  │ net.net.decoder_blocks.0        │ ConvBlock     │  7.0 K │ train │
+│ 85  │ net.net.decoder_blocks.0.conv   │ Sequential    │  7.0 K │ train │
+│ 86  │ net.net.decoder_blocks.0.conv.0 │ Conv2d        │  4.6 K │ train │
+│ 87  │ net.net.decoder_blocks.0.conv.1 │ SyncBatchNorm │     32 │ train │
+│ 88  │ net.net.decoder_blocks.0.conv.2 │ ReLU          │      0 │ train │
+│ 89  │ net.net.decoder_blocks.0.conv.3 │ Conv2d        │  2.3 K │ train │
+│ 90  │ net.net.decoder_blocks.0.conv.4 │ SyncBatchNorm │     32 │ train │
+│ 91  │ net.net.decoder_blocks.0.conv.5 │ ReLU          │      0 │ train │
+│ 92  │ net.net.decoder_blocks.1        │ ConvBlock     │  1.8 K │ train │
+│ 93  │ net.net.decoder_blocks.1.conv   │ Sequential    │  1.8 K │ train │
+│ 94  │ net.net.decoder_blocks.1.conv.0 │ Conv2d        │  1.2 K │ train │
+│ 95  │ net.net.decoder_blocks.1.conv.1 │ SyncBatchNorm │     16 │ train │
+│ 96  │ net.net.decoder_blocks.1.conv.2 │ ReLU          │      0 │ train │
+│ 97  │ net.net.decoder_blocks.1.conv.3 │ Conv2d        │    584 │ train │
+│ 98  │ net.net.decoder_blocks.1.conv.4 │ SyncBatchNorm │     16 │ train │
+│ 99  │ net.net.decoder_blocks.1.conv.5 │ ReLU          │      0 │ train │
+│ 100 │ net.net.decoder_blocks.2        │ ConvBlock     │    456 │ train │
+│ 101 │ net.net.decoder_blocks.2.conv   │ Sequential    │    456 │ train │
+│ 102 │ net.net.decoder_blocks.2.conv.0 │ Conv2d        │    292 │ train │
+│ 103 │ net.net.decoder_blocks.2.conv.1 │ SyncBatchNorm │      8 │ train │
+│ 104 │ net.net.decoder_blocks.2.conv.2 │ ReLU          │      0 │ train │
+│ 105 │ net.net.decoder_blocks.2.conv.3 │ Conv2d        │    148 │ train │
+│ 106 │ net.net.decoder_blocks.2.conv.4 │ SyncBatchNorm │      8 │ train │
+│ 107 │ net.net.decoder_blocks.2.conv.5 │ ReLU          │      0 │ train │
+│ 108 │ net.net.decoder_blocks.3        │ ConvBlock     │    120 │ train │
+│ 109 │ net.net.decoder_blocks.3.conv   │ Sequential    │    120 │ train │
+│ 110 │ net.net.decoder_blocks.3.conv.0 │ Conv2d        │     74 │ train │
+│ 111 │ net.net.decoder_blocks.3.conv.1 │ SyncBatchNorm │      4 │ train │
+│ 112 │ net.net.decoder_blocks.3.conv.2 │ ReLU          │      0 │ train │
+│ 113 │ net.net.decoder_blocks.3.conv.3 │ Conv2d        │     38 │ train │
+│ 114 │ net.net.decoder_blocks.3.conv.4 │ SyncBatchNorm │      4 │ train │
+│ 115 │ net.net.decoder_blocks.3.conv.5 │ ReLU          │      0 │ train │
+│ 116 │ net.net.final_decoder           │ ConvBlock     │    120 │ train │
+│ 117 │ net.net.final_decoder.conv      │ Sequential    │    120 │ train │
+│ 118 │ net.net.final_decoder.conv.0    │ Conv2d        │     74 │ train │
+│ 119 │ net.net.final_decoder.conv.1    │ SyncBatchNorm │      4 │ train │
+│ 120 │ net.net.final_decoder.conv.2    │ ReLU          │      0 │ train │
+│ 121 │ net.net.final_decoder.conv.3    │ Conv2d        │     38 │ train │
+│ 122 │ net.net.final_decoder.conv.4    │ SyncBatchNorm │      4 │ train │
+│ 123 │ net.net.final_decoder.conv.5    │ ReLU          │      0 │ train │
+│ 124 │ net.net.out_conv_R              │ Conv2d        │      3 │ train │
+│ 125 │ net.net.out_conv_S              │ Conv2d        │     17 │ train │
+│ 126 │ rad_denormalization             │ Normalize     │      0 │ train │
+│ 127 │ sat_denormalization             │ Normalize     │      0 │ train │
+└─────┴─────────────────────────────────┴───────────────┴────────┴───────┘
+Trainable params: 30.0 K
+Non-trainable params: 0
+Total params: 30.0 K
+Total estimated model params size (MB): 0
+Modules in train mode: 128
+Modules in eval mode: 0
+Epoch 5:  39%|▍| 190/490 [01:54<03:01,  1.65it/s, v_num=u5_0, train/rad=0.802, train/sat=0.662, train/mse=1.460, val/rad=1.950, val/sat=1.050, 
+/cm/shared/miniconda3/envs/unet/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user.
+  warnings.warn(  # warn only once
+/cm/shared/miniconda3/envs/unet/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('val/rad', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/cm/shared/miniconda3/envs/unet/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('val/sat', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/cm/shared/miniconda3/envs/unet/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('val/mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/cm/shared/miniconda3/envs/unet/lib/python3.10/site-packages/torch/optim/lr_scheduler.py:182: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
+  warnings.warn(
+                                                                                                                                               

checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/files/requirements.txt

@@ -0,0 +1,84 @@
+async-timeout==5.0.1
+annotated-types==0.7.0
+certifi==2025.6.15
+typing_extensions==4.14.0
+nvidia-cuda-runtime-cu12==12.6.77
+packaging==25.0
+pydantic_core==2.33.2
+triton==3.3.1
+sympy==1.14.0
+nvidia-cufft-cu12==11.3.0.4
+charset-normalizer==3.4.2
+nvidia-cusparselt-cu12==0.6.3
+nvidia-cublas-cu12==12.6.4.1
+fsspec==2025.5.1
+pytorch-lightning==2.5.1.post0
+hf-xet==1.1.5
+attrs==25.3.0
+pydantic==2.11.6
+webencodings==0.5.1
+nvidia-cusolver-cu12==11.7.1.2
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+numpy==2.2.6
+torchsummary==1.5.1
+pip==25.1
+idna==3.10
+nvidia-cusparse-cu12==12.5.4.2
+nvidia-cuda-nvrtc-cu12==12.6.77
+setproctitle==1.3.6
+kaggle==1.7.4.5
+Jinja2==3.1.6
+MarkupSafe==3.0.2
+nvidia-nvtx-cu12==12.6.77
+climax==0.3.1
+wheel==0.45.1
+requests==2.32.4
+urllib3==2.4.0
+yarl==1.20.1
+rich==14.0.0
+six==1.17.0
+nvidia-cuda-cupti-cu12==12.6.80
+mpmath==1.3.0
+nvidia-nvjitlink-cu12==12.6.85
+bleach==6.2.0
+torchvision==0.22.1
+gitdb==4.0.12
+aiosignal==1.3.2
+psutil==7.0.0
+multidict==6.4.4
+pillow==11.2.1
+text-unidecode==1.3
+PyYAML==6.0.2
+typing-inspection==0.4.1
+lightning-utilities==0.14.3
+protobuf==6.31.1
+sentry-sdk==2.30.0
+typeshed_client==2.7.0
+smmap==5.0.2
+torchaudio==2.7.1
+tqdm==4.67.1
+wandb==0.20.1
+docstring_parser==0.16
+nvidia-cufile-cu12==1.11.1.6
+aiohttp==3.12.12
+nvidia-nccl-cu12==2.26.2
+mdurl==0.1.2
+huggingface-hub==0.33.0
+filelock==3.18.0
+frozenlist==1.7.0
+networkx==3.4.2
+importlib_resources==6.5.2
+platformdirs==4.3.8
+nvidia-cudnn-cu12==9.5.1.17
+torchmetrics==1.7.3
+torch==2.7.1
+Pygments==2.19.1
+python-slugify==8.0.4
+jsonargparse==4.40.0
+propcache==0.3.2
+GitPython==3.1.44
+markdown-it-py==3.0.0
+setuptools==78.1.1
+nvidia-curand-cu12==10.3.7.77
+click==8.2.1

checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/files/wandb-metadata.json

@@ -0,0 +1,95 @@
+{
+  "os": "Linux-5.4.0-121-generic-x86_64-with-glibc2.31",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-07-05T11:30:57.925618Z",
+  "args": [
+    "--config",
+    "configs/Unet.yaml"
+  ],
+  "program": "/home/vinhbk1/weatherforecast/unet/src/train.py",
+  "codePath": "src/train.py",
+  "email": "weatherforecast1024hcmut@gmail.com",
+  "root": "checkpoint/full/Unet/wandb_logs",
+  "host": "ithndgx005",
+  "executable": "/cm/shared/miniconda3/envs/unet/bin/python",
+  "codePathLocal": "src/train.py",
+  "cpu_count": 128,
+  "cpu_count_logical": 256,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888635834368",
+      "used": "55127412736"
+    }
+  },
+  "memory": {
+    "total": "2164326477824"
+  },
+  "cpu": {
+    "count": 128,
+    "countLogical": 256
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-5e7b14fe-921c-ba36-2aeb-1a5825916bdd"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ae3cfea4-20a5-e014-8450-3d72c9d20c37"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-63b9f644-f11f-db81-3160-40e87114bdfc"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f38d21de-e2a4-9897-5d9a-c6c3fed95c6e"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-8450c54c-f86b-790b-28a4-c33c4d0ecf4b"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-a18dcce2-8b4b-0565-31b4-eb297b1cc4bb"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-46ccbd8d-c84f-32ae-07c3-675928ac6285"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f842f1aa-b36c-40c9-f7ba-095a709c8cb8"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.3"
+}

checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-07-05T18:30:57.834354547+07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmp_1f0yhc0/port-3116289.txt","pid":3116289,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-07-05T18:30:57.835619692+07:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":3116289}
+{"time":"2025-07-05T18:30:57.835577663+07:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":44793,"Zone":""}}
+{"time":"2025-07-05T18:30:57.909455577+07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:53928"}
+{"time":"2025-07-05T18:30:57.928547637+07:00","level":"INFO","msg":"handleInformInit: received","streamId":"zu4qn8u5","id":"127.0.0.1:53928"}
+{"time":"2025-07-05T18:30:58.604632949+07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"zu4qn8u5","id":"127.0.0.1:53928"}
+{"time":"2025-07-05T22:30:30.144946523+07:00","level":"INFO","msg":"Parent process exited, terminating service process."}

checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/logs/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-07-05T18:30:57.959012925+07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/logs/debug-core.log"}
+{"time":"2025-07-05T18:30:58.604600919+07:00","level":"INFO","msg":"stream: created new stream","id":"zu4qn8u5"}
+{"time":"2025-07-05T18:30:58.604628912+07:00","level":"INFO","msg":"stream: started","id":"zu4qn8u5"}
+{"time":"2025-07-05T18:30:58.604636607+07:00","level":"INFO","msg":"writer: Do: started","stream_id":"zu4qn8u5"}
+{"time":"2025-07-05T18:30:58.604650313+07:00","level":"INFO","msg":"sender: started","stream_id":"zu4qn8u5"}
+{"time":"2025-07-05T18:30:58.604686391+07:00","level":"INFO","msg":"handler: started","stream_id":"zu4qn8u5"}
+{"time":"2025-07-05T18:30:58.999782424+07:00","level":"INFO","msg":"Starting system monitor"}

checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/logs/debug.log

@@ -0,0 +1,22 @@
+2025-07-05 18:30:57,686 INFO    MainThread:3116289 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1
+2025-07-05 18:30:57,686 INFO    MainThread:3116289 [wandb_setup.py:_flush():81] Configure stats pid to 3116289
+2025-07-05 18:30:57,686 INFO    MainThread:3116289 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/.config/wandb/settings
+2025-07-05 18:30:57,686 INFO    MainThread:3116289 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/weatherforecast/unet/wandb/settings
+2025-07-05 18:30:57,686 INFO    MainThread:3116289 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-07-05 18:30:57,686 INFO    MainThread:3116289 [wandb_init.py:setup_run_log_directory():703] Logging user logs to checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/logs/debug.log
+2025-07-05 18:30:57,686 INFO    MainThread:3116289 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/logs/debug-internal.log
+2025-07-05 18:30:57,686 INFO    MainThread:3116289 [wandb_init.py:init():831] calling init triggers
+2025-07-05 18:30:57,686 INFO    MainThread:3116289 [wandb_init.py:init():836] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-05 18:30:57,686 INFO    MainThread:3116289 [wandb_init.py:init():872] starting backend
+2025-07-05 18:30:57,909 INFO    MainThread:3116289 [wandb_init.py:init():875] sending inform_init request
+2025-07-05 18:30:57,925 INFO    MainThread:3116289 [wandb_init.py:init():883] backend started and connected
+2025-07-05 18:30:57,926 INFO    MainThread:3116289 [wandb_init.py:init():956] updated telemetry
+2025-07-05 18:30:57,927 INFO    MainThread:3116289 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout
+2025-07-05 18:30:58,949 INFO    MainThread:3116289 [wandb_init.py:init():1032] starting run threads in backend
+2025-07-05 18:30:59,105 INFO    MainThread:3116289 [wandb_run.py:_console_start():2453] atexit reg
+2025-07-05 18:30:59,105 INFO    MainThread:3116289 [wandb_run.py:_redirect():2301] redirect: wrap_raw
+2025-07-05 18:30:59,105 INFO    MainThread:3116289 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2025-07-05 18:30:59,106 INFO    MainThread:3116289 [wandb_run.py:_redirect():2393] Redirects installed.
+2025-07-05 18:30:59,117 INFO    MainThread:3116289 [wandb_init.py:init():1078] run started, returning control to user process
+2025-07-05 19:01:44,657 INFO    MainThread:3116289 [wandb_run.py:_config_callback():1358] config_cb None None {'pretrained_path': '', 'lr': 0.0005, 'beta_1': 0.9, 'beta_2': 0.99, 'weight_decay': 1e-05, 'warmup_epochs': 10, 'max_epochs': 50, 'warmup_start_lr': 1e-08, 'eta_min': 1e-08, '_instantiator': 'pytorch_lightning.cli.instantiate_module', 'dir_data': '/cm/archive/vinhbk1/NhaBe', 'batch_size': 64, 'hours_predicted': 3, 'num_workers': 8, 'pin_memory': False, 'time_points_rad': 1, 'time_points_sat': 1, 'sat_inp_vars': 'total_precipitation', 'sat_out_vars': 'total_precipitation', 'sat_size': 25, 'rad_inp_vars': 'precipitation', 'rad_out_vars': 'precipitation', 'rad_size': 400, 'ablation': 'full'}

checkpoint/full/Unet/wandb_logs/wandb/run-20250705_183057-zu4qn8u5/run-zu4qn8u5.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:db3a615026366902dcd0eefb5a396b92da6ec781c36dc27469c74a02c267c9d0
+size 4587520

checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/files/requirements.txt

@@ -0,0 +1,84 @@
+async-timeout==5.0.1
+annotated-types==0.7.0
+certifi==2025.6.15
+typing_extensions==4.14.0
+nvidia-cuda-runtime-cu12==12.6.77
+packaging==25.0
+pydantic_core==2.33.2
+triton==3.3.1
+sympy==1.14.0
+nvidia-cufft-cu12==11.3.0.4
+charset-normalizer==3.4.2
+nvidia-cusparselt-cu12==0.6.3
+nvidia-cublas-cu12==12.6.4.1
+fsspec==2025.5.1
+pytorch-lightning==2.5.1.post0
+hf-xet==1.1.5
+attrs==25.3.0
+pydantic==2.11.6
+webencodings==0.5.1
+nvidia-cusolver-cu12==11.7.1.2
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+numpy==2.2.6
+torchsummary==1.5.1
+pip==25.1
+idna==3.10
+nvidia-cusparse-cu12==12.5.4.2
+nvidia-cuda-nvrtc-cu12==12.6.77
+setproctitle==1.3.6
+kaggle==1.7.4.5
+Jinja2==3.1.6
+MarkupSafe==3.0.2
+nvidia-nvtx-cu12==12.6.77
+climax==0.3.1
+wheel==0.45.1
+requests==2.32.4
+urllib3==2.4.0
+yarl==1.20.1
+rich==14.0.0
+six==1.17.0
+nvidia-cuda-cupti-cu12==12.6.80
+mpmath==1.3.0
+nvidia-nvjitlink-cu12==12.6.85
+bleach==6.2.0
+torchvision==0.22.1
+gitdb==4.0.12
+aiosignal==1.3.2
+psutil==7.0.0
+multidict==6.4.4
+pillow==11.2.1
+text-unidecode==1.3
+PyYAML==6.0.2
+typing-inspection==0.4.1
+lightning-utilities==0.14.3
+protobuf==6.31.1
+sentry-sdk==2.30.0
+typeshed_client==2.7.0
+smmap==5.0.2
+torchaudio==2.7.1
+tqdm==4.67.1
+wandb==0.20.1
+docstring_parser==0.16
+nvidia-cufile-cu12==1.11.1.6
+aiohttp==3.12.12
+nvidia-nccl-cu12==2.26.2
+mdurl==0.1.2
+huggingface-hub==0.33.0
+filelock==3.18.0
+frozenlist==1.7.0
+networkx==3.4.2
+importlib_resources==6.5.2
+platformdirs==4.3.8
+nvidia-cudnn-cu12==9.5.1.17
+torchmetrics==1.7.3
+torch==2.7.1
+Pygments==2.19.1
+python-slugify==8.0.4
+jsonargparse==4.40.0
+propcache==0.3.2
+GitPython==3.1.44
+markdown-it-py==3.0.0
+setuptools==78.1.1
+nvidia-curand-cu12==10.3.7.77
+click==8.2.1

checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/files/wandb-metadata.json

@@ -0,0 +1,95 @@
+{
+  "os": "Linux-5.4.0-121-generic-x86_64-with-glibc2.31",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-07-07T01:52:43.628059Z",
+  "args": [
+    "--config",
+    "configs/Unet.yaml"
+  ],
+  "program": "/home/vinhbk1/weatherforecast/unet/src/train.py",
+  "codePath": "src/train.py",
+  "email": "weatherforecast1024hcmut@gmail.com",
+  "root": "checkpoint/full/Unet/wandb_logs",
+  "host": "ithndgx005",
+  "executable": "/cm/shared/miniconda3/envs/unet/bin/python",
+  "codePathLocal": "src/train.py",
+  "cpu_count": 128,
+  "cpu_count_logical": 256,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888635834368",
+      "used": "55265112064"
+    }
+  },
+  "memory": {
+    "total": "2164326477824"
+  },
+  "cpu": {
+    "count": 128,
+    "countLogical": 256
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-5e7b14fe-921c-ba36-2aeb-1a5825916bdd"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ae3cfea4-20a5-e014-8450-3d72c9d20c37"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-63b9f644-f11f-db81-3160-40e87114bdfc"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f38d21de-e2a4-9897-5d9a-c6c3fed95c6e"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-8450c54c-f86b-790b-28a4-c33c4d0ecf4b"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-a18dcce2-8b4b-0565-31b4-eb297b1cc4bb"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-46ccbd8d-c84f-32ae-07c3-675928ac6285"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f842f1aa-b36c-40c9-f7ba-095a709c8cb8"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.3"
+}

checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/logs/debug-core.log

@@ -0,0 +1,7 @@
+{"time":"2025-07-07T08:52:43.45418796+07:00","level":"INFO","msg":"main: starting server","port-filename":"/tmp/tmpijmwtp6t/port-2461292.txt","pid":2461292,"log-level":0,"disable-analytics":false,"shutdown-on-parent-exit":false,"enable-dcgm-profiling":false}
+{"time":"2025-07-07T08:52:43.455579093+07:00","level":"INFO","msg":"Will exit if parent process dies.","ppid":2461292}
+{"time":"2025-07-07T08:52:43.455593571+07:00","level":"INFO","msg":"server is running","addr":{"IP":"127.0.0.1","Port":34417,"Zone":""}}
+{"time":"2025-07-07T08:52:43.591946298+07:00","level":"INFO","msg":"connection: ManageConnectionData: new connection created","id":"127.0.0.1:34816"}
+{"time":"2025-07-07T08:52:43.630590572+07:00","level":"INFO","msg":"handleInformInit: received","streamId":"ou1t8ei4","id":"127.0.0.1:34816"}
+{"time":"2025-07-07T08:52:44.462356565+07:00","level":"INFO","msg":"handleInformInit: stream started","streamId":"ou1t8ei4","id":"127.0.0.1:34816"}
+{"time":"2025-07-07T09:23:37.340942072+07:00","level":"INFO","msg":"Parent process exited, terminating service process."}

checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/logs/debug-internal.log

@@ -0,0 +1,8 @@
+{"time":"2025-07-07T08:52:43.786989998+07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/logs/debug-core.log"}
+{"time":"2025-07-07T08:52:44.462305768+07:00","level":"INFO","msg":"stream: created new stream","id":"ou1t8ei4"}
+{"time":"2025-07-07T08:52:44.4623492+07:00","level":"INFO","msg":"stream: started","id":"ou1t8ei4"}
+{"time":"2025-07-07T08:52:44.462346996+07:00","level":"INFO","msg":"handler: started","stream_id":"ou1t8ei4"}
+{"time":"2025-07-07T08:52:44.462369218+07:00","level":"INFO","msg":"sender: started","stream_id":"ou1t8ei4"}
+{"time":"2025-07-07T08:52:44.462364569+07:00","level":"INFO","msg":"writer: Do: started","stream_id":"ou1t8ei4"}
+{"time":"2025-07-07T08:52:45.010883933+07:00","level":"INFO","msg":"Starting system monitor"}
+{"time":"2025-07-07T08:52:45.477455488+07:00","level":"ERROR","msg":"file transfer: upload: failed to upload: 400 Bad Request","task":"DefaultUploadTask{FileKind: 1, Path: checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/files/wandb-metadata.json, Name: wandb-metadata.json, Url: https://storage.googleapis.com/wandb-production.appspot.com/weatherforecast1024/NhaBe/ou1t8ei4/wandb-metadata.json?X-Goog-Algorithm=GOOG4-RSA-SHA256&X-Goog-Credential=gorilla-files-url-signer-man%40wandb-production.iam.gserviceaccount.com%2F20250707%2Fauto%2Fstorage%2Fgoog4_request&X-Goog-Date=20250707T015245Z&X-Goog-Expires=86399&X-Goog-Signature=048f05e8bdb2838e0fb7b05d69cc870b2a36f2196c0793d100fe1b18cf87643c4598200ef38ae8125510c15f1b4c87bc21e8ab12eb20885f8b6a93a785da00afdca79046efe1f94cb10b7dfc7a2fdb63d214f4ab8a06be67c0f01612f11fdf4181055615061117a5355978b64a3d94ef0c8cb67009554acf84e80f99906cd78e0a9a02476e4f1078a2d525ce72d3da48c6673f0fb6e7a0f03db4f3c74140aad9db78dd711dfc2b6af315e2f44a9ed2ee7b36fbd593a730acb26aa9ccdb1d3c0c9d0f9fc37eb8f0e1abc3ce1a2e1d4929abb71f26e3884cb776f8f16fefa9d01bd0292bf6fefca867079a1db1fff53f4be9828072c2b5c55ef0b2631739a2df18&X-Goog-SignedHeaders=host&X-User=weatherforecast1024hcmut, Size: 502}"}

checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/logs/debug.log

@@ -0,0 +1,21 @@
+2025-07-07 08:52:43,118 INFO    MainThread:2461292 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1
+2025-07-07 08:52:43,119 INFO    MainThread:2461292 [wandb_setup.py:_flush():81] Configure stats pid to 2461292
+2025-07-07 08:52:43,119 INFO    MainThread:2461292 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/.config/wandb/settings
+2025-07-07 08:52:43,119 INFO    MainThread:2461292 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/weatherforecast/unet/wandb/settings
+2025-07-07 08:52:43,159 INFO    MainThread:2461292 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-07-07 08:52:43,160 INFO    MainThread:2461292 [wandb_init.py:setup_run_log_directory():703] Logging user logs to checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/logs/debug.log
+2025-07-07 08:52:43,160 INFO    MainThread:2461292 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/logs/debug-internal.log
+2025-07-07 08:52:43,169 INFO    MainThread:2461292 [wandb_init.py:init():831] calling init triggers
+2025-07-07 08:52:43,170 INFO    MainThread:2461292 [wandb_init.py:init():836] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-07-07 08:52:43,173 INFO    MainThread:2461292 [wandb_init.py:init():872] starting backend
+2025-07-07 08:52:43,592 INFO    MainThread:2461292 [wandb_init.py:init():875] sending inform_init request
+2025-07-07 08:52:43,627 INFO    MainThread:2461292 [wandb_init.py:init():883] backend started and connected
+2025-07-07 08:52:43,629 INFO    MainThread:2461292 [wandb_init.py:init():956] updated telemetry
+2025-07-07 08:52:43,629 INFO    MainThread:2461292 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout
+2025-07-07 08:52:44,918 INFO    MainThread:2461292 [wandb_init.py:init():1032] starting run threads in backend
+2025-07-07 08:52:45,138 INFO    MainThread:2461292 [wandb_run.py:_console_start():2453] atexit reg
+2025-07-07 08:52:45,138 INFO    MainThread:2461292 [wandb_run.py:_redirect():2301] redirect: wrap_raw
+2025-07-07 08:52:45,144 INFO    MainThread:2461292 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2025-07-07 08:52:45,151 INFO    MainThread:2461292 [wandb_run.py:_redirect():2393] Redirects installed.
+2025-07-07 08:52:45,166 INFO    MainThread:2461292 [wandb_init.py:init():1078] run started, returning control to user process

checkpoint/full/Unet/wandb_logs/wandb/run-20250707_085242-ou1t8ei4/run-ou1t8ei4.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fdf95756b942a79ba39fc2df2c4495da845208691c7ca035ee41ed0328190f97
+size 393216

checkpoint/no/AttR2Unet/checkpoints/epoch_011.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:66c0388331c3b367c5c212a8bc6080535f4fb49551a233bfdd2601337741e884
+size 593623

checkpoint/no/AttR2Unet/checkpoints/last.ckpt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4d897ef0d589ca06b78a20d6f13b05750902e20f31370ca459866e7c9d0cd5cb
+size 593623

checkpoint/no/AttR2Unet/csv_logs/version_0/hparams.yaml

@@ -0,0 +1,24 @@
+_instantiator: pytorch_lightning.cli.instantiate_module
+ablation: 'no'
+batch_size: 64
+beta_1: 0.9
+beta_2: 0.99
+dir_data: /cm/archive/vinhbk1/NhaBe
+eta_min: 1.0e-08
+hours_predicted: 3
+lr: 0.0005
+max_epochs: 50
+num_workers: 8
+pin_memory: false
+pretrained_path: ''
+rad_inp_vars: precipitation
+rad_out_vars: precipitation
+rad_size: 400
+sat_inp_vars: total_precipitation
+sat_out_vars: total_precipitation
+sat_size: 25
+time_points_rad: 1
+time_points_sat: 1
+warmup_epochs: 10
+warmup_start_lr: 1.0e-08
+weight_decay: 1.0e-05

checkpoint/no/AttR2Unet/csv_logs/version_0/metrics.csv

@@ -0,0 +1,347 @@
+epoch,lr-AdamW/pg1,lr-AdamW/pg2,step,train/mse,train/rad,train/sat,val/mse,val/rad,val/sat
+,7.806511533866747e-07,7.806511533866747e-07,49,,,,,,
+0,,,49,2.485987424850464,1.4775511026382446,1.0084363222122192,,,
+,0.0004401026893701954,0.0004401026893701954,99,,,,,,
+0,,,99,1.271034598350525,0.7922484874725342,0.47878608107566833,,,
+,0.0002303906183635172,0.0002303906183635172,149,,,,,,
+0,,,149,2.6905345916748047,1.4208334684371948,1.2697011232376099,,,
+,8.764623515769576e-05,8.764623515769576e-05,199,,,,,,
+0,,,199,1.8767162561416626,0.9513498544692993,0.9253664016723633,,,
+,0.0004992293488466136,0.0004992293488466136,249,,,,,,
+0,,,249,1.9308807849884033,1.1979695558547974,0.7329112887382507,,,
+,5.990731062982018e-05,5.990731062982018e-05,299,,,,,,
+0,,,299,2.469799757003784,1.155877947807312,1.3139218091964722,,,
+,0.0002696193816364834,0.0002696193816364834,349,,,,,,
+0,,,349,1.9560089111328125,1.177018165588379,0.7789908051490784,,,
+,0.0004123637648423488,0.0004123637648423488,399,,,,,,
+0,,,399,2.1415023803710938,1.2166975736618042,0.9248047471046448,,,
+,7.806511533867857e-07,7.806511533867857e-07,449,,,,,,
+0,,,449,2.033639430999756,1.2084654569625854,0.8251739740371704,,,
+0,,,487,,,,3.220388412475586,2.0765323638916016,1.143856167793274
+,0.0004401026893701956,0.0004401026893701956,499,,,,,,
+1,,,499,2.016099452972412,0.8705683350563049,1.145531177520752,,,
+,0.00023039061836352563,0.00023039061836352563,549,,,,,,
+1,,,549,1.9373950958251953,0.9521660208702087,0.9852290749549866,,,
+,8.764623515769884e-05,8.764623515769884e-05,599,,,,,,
+1,,,599,2.4021716117858887,1.1236568689346313,1.2785148620605469,,,
+,0.0004992293488466312,0.0004992293488466312,649,,,,,,
+1,,,649,1.6054373979568481,0.69674152135849,0.9086958765983582,,,
+,5.990731062982157e-05,5.990731062982157e-05,699,,,,,,
+1,,,699,2.2697815895080566,1.2143832445144653,1.0553984642028809,,,
+,0.00026961938163649353,0.00026961938163649353,749,,,,,,
+1,,,749,2.773038387298584,1.6368024349212646,1.1362359523773193,,,
+,0.0004123637648423198,0.0004123637648423198,799,,,,,,
+1,,,799,2.3351945877075195,1.337319254875183,0.9978753924369812,,,
+,7.806511533868971e-07,7.806511533868971e-07,849,,,,,,
+1,,,849,2.1499857902526855,1.1353747844696045,1.0146108865737915,,,
+,0.00044010268937021277,0.00044010268937021277,899,,,,,,
+1,,,899,2.0597612857818604,1.1860464811325073,0.873714804649353,,,
+,0.00023039061836349384,0.00023039061836349384,949,,,,,,
+1,,,949,1.243640422821045,0.4815954566001892,0.7620449066162109,,,
+1,,,975,,,,3.1796762943267822,2.0348591804504395,1.1448172330856323
+,8.764623515768775e-05,8.764623515768775e-05,999,,,,,,
+2,,,999,1.6237330436706543,0.6219517588615417,1.0017813444137573,,,
+,0.0004992293488466488,0.0004992293488466488,1049,,,,,,
+2,,,1049,2.267584800720215,1.3452175855636597,0.9223671555519104,,,
+,5.990731062981273e-05,5.990731062981273e-05,1099,,,,,,
+2,,,1099,2.1042823791503906,1.2009294033050537,0.9033530950546265,,,
+,0.0002696193816364547,0.0002696193816364547,1149,,,,,,
+2,,,1149,1.2282277345657349,0.5309398174285889,0.697287917137146,,,
+,0.0004123637648423333,0.0004123637648423333,1199,,,,,,
+2,,,1199,1.258795142173767,0.6327914595603943,0.6260036826133728,,,
+,7.806511533868142e-07,7.806511533868142e-07,1249,,,,,,
+2,,,1249,1.8212740421295166,0.8864027261734009,0.934871256351471,,,
+,0.000440102689370307,0.000440102689370307,1299,,,,,,
+2,,,1299,1.8859176635742188,0.9280247688293457,0.9578928351402283,,,
+,0.00023039061836354518,0.00023039061836354518,1349,,,,,,
+2,,,1349,1.9002772569656372,0.5404709577560425,1.3598062992095947,,,
+,8.764623515770832e-05,8.764623515770832e-05,1399,,,,,,
+2,,,1399,1.5460271835327148,0.6763516664505005,0.8696755766868591,,,
+,0.0004992293488467564,0.0004992293488467564,1449,,,,,,
+2,,,1449,1.4749705791473389,0.9029025435447693,0.5720679759979248,,,
+2,,,1463,,,,3.2163302898406982,2.0280601978302,1.188269853591919
+,5.9907310629826644e-05,5.9907310629826644e-05,1499,,,,,,
+3,,,1499,1.957237958908081,0.9719163775444031,0.985321581363678,,,
+,0.0002696193816365116,0.0002696193816365116,1549,,,,,,
+3,,,1549,2.2931954860687256,1.2943482398986816,0.998847246170044,,,
+,0.0004123637648422745,0.0004123637648422745,1599,,,,,,
+3,,,1599,2.4210193157196045,1.1082857847213745,1.31273353099823,,,
+,7.806511533863131e-07,7.806511533863131e-07,1649,,,,,,
+3,,,1649,1.6625289916992188,0.6846383810043335,0.97789067029953,,,
+,0.00044010268937007323,0.00044010268937007323,1699,,,,,,
+3,,,1699,1.4047675132751465,0.7306445240974426,0.6741230487823486,,,
+,0.0002303906183635893,0.0002303906183635893,1749,,,,,,
+3,,,1749,1.730273962020874,0.8566880822181702,0.8735859394073486,,,
+,8.764623515772343e-05,8.764623515772343e-05,1799,,,,,,
+3,,,1799,1.776611328125,0.9709169864654541,0.8056943416595459,,,
+,0.0004992293488466852,0.0004992293488466852,1849,,,,,,
+3,,,1849,3.3605189323425293,1.640255093574524,1.7202638387680054,,,
+,5.990731062981895e-05,5.990731062981895e-05,1899,,,,,,
+3,,,1899,1.9088561534881592,0.8606491684913635,1.0482069253921509,,,
+,0.00026961938163647467,0.00026961938163647467,1949,,,,,,
+3,,,1949,1.816565752029419,0.9718300700187683,0.8447357416152954,,,
+3,,,1951,,,,3.2185006141662598,2.062351942062378,1.1561487913131714
+,0.00041236376484221106,0.00041236376484221106,1999,,,,,,
+4,,,1999,2.0659022331237793,1.0408923625946045,1.0250097513198853,,,
+,7.80651153386647e-07,7.80651153386647e-07,2049,,,,,,
+4,,,2049,1.8791499137878418,0.9867289066314697,0.8924210667610168,,,
+,0.00044010268937000877,0.00044010268937000877,2099,,,,,,
+4,,,2099,2.6083688735961914,1.308364987373352,1.3000038862228394,,,
+,0.00023039061836354922,0.00023039061836354922,2149,,,,,,
+4,,,2149,1.8601658344268799,1.0010020732879639,0.859163761138916,,,
+,8.764623515770644e-05,8.764623515770644e-05,2199,,,,,,
+4,,,2199,2.1352200508117676,0.8846878409385681,1.2505321502685547,,,
+,0.0004992293488469725,0.0004992293488469725,2249,,,,,,
+4,,,2249,1.9601950645446777,0.9984703063964844,0.9617247581481934,,,
+,5.990731062984982e-05,5.990731062984982e-05,2299,,,,,,
+4,,,2299,2.172466278076172,1.2651060819625854,0.907360315322876,,,
+,0.00026961938163662873,0.00026961938163662873,2349,,,,,,
+4,,,2349,1.1063743829727173,0.5499252676963806,0.5564491152763367,,,
+,0.0004123637648424494,0.0004123637648424494,2399,,,,,,
+4,,,2399,2.28848934173584,1.4900113344192505,0.7984780073165894,,,
+4,,,2439,,,,3.7320096492767334,2.470048189163208,1.2619614601135254
+,7.806511533869514e-07,7.806511533869514e-07,2449,,,,,,
+5,,,2449,1.6565027236938477,0.9683687090873718,0.6881340742111206,,,
+,0.0004401026893702612,0.0004401026893702612,2499,,,,,,
+5,,,2499,2.073665142059326,1.2118834257125854,0.8617817163467407,,,
+,0.0002303906183635175,0.0002303906183635175,2549,,,,,,
+5,,,2549,1.1273250579833984,0.44634079933166504,0.6809842586517334,,,
+,8.76462351576981e-05,8.76462351576981e-05,2599,,,,,,
+5,,,2599,1.4989230632781982,0.9152175784111023,0.5837055444717407,,,
+,0.0004992293488468831,0.0004992293488468831,2649,,,,,,
+5,,,2649,1.9564077854156494,1.1899816989898682,0.766426146030426,,,
+,5.990731062984678e-05,5.990731062984678e-05,2699,,,,,,
+5,,,2699,1.8755183219909668,0.8826121091842651,0.9929062128067017,,,
+,0.0002696193816365957,0.0002696193816365957,2749,,,,,,
+5,,,2749,1.9588117599487305,1.2094916105270386,0.7493200898170471,,,
+,0.00041236376484268307,0.00041236376484268307,2799,,,,,,
+5,,,2799,2.422802448272705,1.3581856489181519,1.0646167993545532,,,
+,7.806511533878403e-07,7.806511533878403e-07,2849,,,,,,
+5,,,2849,2.1674697399139404,1.3016526699066162,0.8658170700073242,,,
+,0.0004401026893701968,0.0004401026893701968,2899,,,,,,
+5,,,2899,1.7519019842147827,0.8129711747169495,0.9389308094978333,,,
+5,,,2927,,,,3.3037784099578857,2.0859692096710205,1.2178090810775757
+,0.00023039061836364468,0.00023039061836364468,2949,,,,,,
+6,,,2949,1.6902868747711182,0.8633924722671509,0.8268944025039673,,,
+,8.764623515774753e-05,8.764623515774753e-05,2999,,,,,,
+6,,,2999,2.2631747722625732,1.3868203163146973,0.8763543963432312,,,
+,0.0004992293488468112,0.0004992293488468112,3049,,,,,,
+6,,,3049,2.2217278480529785,0.9590405225753784,1.2626874446868896,,,
+,5.990731062987761e-05,5.990731062987761e-05,3099,,,,,,
+6,,,3099,2.154656410217285,0.8105692267417908,1.3440871238708496,,,
+,0.0002696193816367422,0.0002696193816367422,3149,,,,,,
+6,,,3149,1.290679693222046,0.5537689328193665,0.7369107007980347,,,
+,0.00041236376484233314,0.00041236376484233314,3199,,,,,,
+6,,,3199,1.8764898777008057,0.7580167651176453,1.1184731721878052,,,
+,7.806511533865081e-07,7.806511533865081e-07,3249,,,,,,
+6,,,3249,1.8068757057189941,1.0319641828536987,0.7749115228652954,,,
+,0.00044010268937044955,0.00044010268937044955,3299,,,,,,
+6,,,3299,2.014021158218384,1.0756416320800781,0.9383795857429504,,,
+,0.0002303906183634541,0.0002303906183634541,3349,,,,,,
+6,,,3349,1.7657231092453003,0.9542507529258728,0.8114723563194275,,,
+,8.764623515767607e-05,8.764623515767607e-05,3399,,,,,,
+6,,,3399,2.670576333999634,0.9480571150779724,1.7225192785263062,,,
+6,,,3415,,,,3.382279396057129,2.1507346630096436,1.2315443754196167
+,0.000499229348846739,0.000499229348846739,3449,,,,,,
+7,,,3449,2.097353458404541,1.1357287168502808,0.9616246819496155,,,
+,5.990731062978148e-05,5.990731062978148e-05,3499,,,,,,
+7,,,3499,2.112830638885498,1.048595905303955,1.064234733581543,,,
+,0.00026961938163631816,0.00026961938163631816,3549,,,,,,
+7,,,3549,1.3705108165740967,0.6781675815582275,0.6923432946205139,,,
+,0.0004123637648422545,0.0004123637648422545,3599,,,,,,
+7,,,3599,1.4182839393615723,0.7291467785835266,0.6891371607780457,,,
+,7.806511533868135e-07,7.806511533868135e-07,3649,,,,,,
+7,,,3649,1.557459831237793,0.8546115159988403,0.7028483152389526,,,
+,0.00044010268937038537,0.00044010268937038537,3699,,,,,,
+7,,,3699,1.9530367851257324,0.9142968654632568,1.0387399196624756,,,
+,0.00023039061836375418,0.00023039061836375418,3749,,,,,,
+7,,,3749,1.942488431930542,1.0580037832260132,0.8844846487045288,,,
+,8.764623515778853e-05,8.764623515778853e-05,3799,,,,,,
+7,,,3799,1.7513595819473267,0.8610121607780457,0.890347421169281,,,
+,0.0004992293488466671,0.0004992293488466671,3849,,,,,,
+7,,,3849,1.528515338897705,0.7646268606185913,0.7638884782791138,,,
+,5.990731062986011e-05,5.990731062986011e-05,3899,,,,,,
+7,,,3899,1.7800904512405396,0.8208164572715759,0.9592739939689636,,,
+7,,,3903,,,,3.275517702102661,2.132962703704834,1.142554521560669
+,0.000269619381636652,0.000269619381636652,3949,,,,,,
+8,,,3949,2.3220577239990234,1.5505077838897705,0.7715499401092529,,,
+,0.00041236376484219626,0.00041236376484219626,3999,,,,,,
+8,,,3999,1.8172638416290283,0.9555058479309082,0.8617579936981201,,,
+,7.806511533865915e-07,7.806511533865915e-07,4049,,,,,,
+8,,,4049,2.2480082511901855,1.090814471244812,1.157193899154663,,,
+,0.0004401026893703302,0.0004401026893703302,4099,,,,,,
+8,,,4099,2.1123478412628174,1.1279720067977905,0.9843757748603821,,,
+,0.00023039061836370835,0.00023039061836370835,4149,,,,,,
+8,,,4149,2.7684669494628906,1.7307007312774658,1.0377663373947144,,,
+,8.764623515777477e-05,8.764623515777477e-05,4199,,,,,,
+8,,,4199,1.2539583444595337,0.507749080657959,0.7462092638015747,,,
+,0.0004992293488465951,0.0004992293488465951,4249,,,,,,
+8,,,4249,1.8466356992721558,0.9845156073570251,0.8621200919151306,,,
+,5.990731062985243e-05,5.990731062985243e-05,4299,,,,,,
+8,,,4299,1.9950793981552124,0.9259144067764282,1.0691649913787842,,,
+,0.0002696193816366258,0.0002696193816366258,4349,,,,,,
+8,,,4349,1.1099203824996948,0.5030733942985535,0.6068469882011414,,,
+8,,,4391,,,,3.136686325073242,1.9866657257080078,1.1500205993652344
+,0.0004123637648421378,0.0004123637648421378,4399,,,,,,
+9,,,4399,2.047489643096924,1.0292972326278687,1.0181924104690552,,,
+,7.806511533863697e-07,7.806511533863697e-07,4449,,,,,,
+9,,,4449,2.456502914428711,1.2804772853851318,1.176025629043579,,,
+,0.0004401026893702407,0.0004401026893702407,4499,,,,,,
+9,,,4499,2.43601655960083,1.7680538892745972,0.6679627895355225,,,
+,0.0002303906183636908,0.0002303906183636908,4549,,,,,,
+9,,,4549,2.1891915798187256,1.2218468189239502,0.9673447012901306,,,
+,8.764623515776115e-05,8.764623515776115e-05,4599,,,,,,
+9,,,4599,1.7026989459991455,0.7146903872489929,0.9880085587501526,,,
+,0.0004992293488472418,0.0004992293488472418,4649,,,,,,
+9,,,4649,1.9249037504196167,1.0918080806732178,0.8330956697463989,,,
+,5.990731062983553e-05,5.990731062983553e-05,4699,,,,,,
+9,,,4699,1.7635793685913086,1.1857472658157349,0.5778321623802185,,,
+,0.00026961938163658553,0.00026961938163658553,4749,,,,,,
+9,,,4749,2.9617323875427246,1.3254189491271973,1.6363133192062378,,,
+,0.00041236376484267363,0.00041236376484267363,4799,,,,,,
+9,,,4799,1.6199729442596436,0.8111218810081482,0.8088510632514954,,,
+,7.806511533872576e-07,7.806511533872576e-07,4849,,,,,,
+9,,,4849,1.8316630125045776,0.9401909708976746,0.8914720416069031,,,
+9,,,4879,,,,3.15055513381958,1.97207772731781,1.1784772872924805
+,0.000440102689370186,0.000440102689370186,4899,,,,,,
+10,,,4899,2.0796468257904053,1.254194736480713,0.8254521489143372,,,
+,0.00023039061836363658,0.00023039061836363658,4949,,,,,,
+10,,,4949,1.764545202255249,0.9484091401100159,0.8161361217498779,,,
+,8.76462351577442e-05,8.76462351577442e-05,4999,,,,,,
+10,,,4999,2.105269432067871,1.0303531885147095,1.074916124343872,,,
+,0.0004992293488464517,0.0004992293488464517,5049,,,,,,
+10,,,5049,1.3775943517684937,0.6233800053596497,0.754214346408844,,,
+,5.9907310629837086e-05,5.9907310629837086e-05,5099,,,,,,
+10,,,5099,1.9358415603637695,0.9653550386428833,0.970486581325531,,,
+,0.00026961938163654525,0.00026961938163654525,5149,,,,,,
+10,,,5149,2.0367302894592285,0.46001511812210083,1.5767152309417725,,,
+,0.00041236376484201037,0.00041236376484201037,5199,,,,,,
+10,,,5199,1.9343938827514648,1.0853440761566162,0.8490498065948486,,,
+,7.806511533858972e-07,7.806511533858972e-07,5249,,,,,,
+10,,,5249,1.5506234169006348,0.7967603206634521,0.7538630962371826,,,
+,0.0004401026893701207,0.0004401026893701207,5299,,,,,,
+10,,,5299,1.6738715171813965,0.7312909960746765,0.94258052110672,,,
+,0.00023039061836361898,0.00023039061836361898,5349,,,,,,
+10,,,5349,2.2042834758758545,1.2980761528015137,0.9062073230743408,,,
+10,,,5367,,,,3.136448383331299,1.97783625125885,1.1586120128631592
+,8.764623515773041e-05,8.764623515773041e-05,5399,,,,,,
+11,,,5399,1.464619755744934,0.8699076771736145,0.5947120785713196,,,
+,0.000499229348847099,0.000499229348847099,5449,,,,,,
+11,,,5449,1.814762830734253,0.5987300872802734,1.2160327434539795,,,
+,5.9907310629820186e-05,5.9907310629820186e-05,5499,,,,,,
+11,,,5499,2.1400108337402344,1.1068135499954224,1.0331974029541016,,,
+,0.000269619381636519,0.000269619381636519,5549,,,,,,
+11,,,5549,2.442185640335083,1.4643326997756958,0.977853000164032,,,
+,0.0004123637648431512,0.0004123637648431512,5599,,,,,,
+11,,,5599,1.6475250720977783,0.8550796508789062,0.7924454212188721,,,
+,7.806511533890063e-07,7.806511533890063e-07,5649,,,,,,
+11,,,5649,1.471982479095459,0.644266664981842,0.8277158141136169,,,
+,0.00044010268937069074,0.00044010268937069074,5699,,,,,,
+11,,,5699,2.334575653076172,1.414272665977478,0.9203029870986938,,,
+,0.00023039061836355908,0.00023039061836355908,5749,,,,,,
+11,,,5749,1.6169171333312988,0.7774771451950073,0.8394399285316467,,,
+,8.764623515771671e-05,8.764623515771671e-05,5799,,,,,,
+11,,,5799,1.8048462867736816,0.9759384393692017,0.8289077877998352,,,
+,0.0004992293488462894,0.0004992293488462894,5849,,,,,,
+11,,,5849,3.2447726726531982,1.9477839469909668,1.2969887256622314,,,
+11,,,5855,,,,3.119295358657837,1.9720841646194458,1.1472110748291016
+,5.990731062990803e-05,5.990731062990803e-05,5899,,,,,,
+12,,,5899,1.8285789489746094,0.9557724595069885,0.8728064298629761,,,
+,0.0002696193816368533,0.0002696193816368533,5949,,,,,,
+12,,,5949,1.665055751800537,0.7297932505607605,0.9352625608444214,,,
+,0.00041236376484249886,0.00041236376484249886,5999,,,,,,
+12,,,5999,1.8890719413757324,0.8400761485099792,1.048995852470398,,,
+,7.806511533876736e-07,7.806511533876736e-07,6049,,,,,,
+12,,,6049,2.157853603363037,1.2722495794296265,0.8856040835380554,,,
+,0.00044010268937000167,0.00044010268937000167,6099,,,,,,
+12,,,6099,2.3558623790740967,1.0877281427383423,1.2681342363357544,,,
+,0.00023039061836354146,0.00023039061836354146,6149,,,,,,
+12,,,6149,2.225086212158203,1.2112278938293457,1.0138583183288574,,,
+,8.764623515771383e-05,8.764623515771383e-05,6199,,,,,,
+12,,,6199,0.9583333730697632,0.5232091546058655,0.4351242184638977,,,
+,0.0004992293488469365,0.0004992293488469365,6249,,,,,,
+12,,,6249,1.7786357402801514,0.8889786601066589,0.8896570801734924,,,
+,5.990731062987974e-05,5.990731062987974e-05,6299,,,,,,
+12,,,6299,1.731277346611023,0.994939923286438,0.736337423324585,,,
+12,,,6343,,,,3.2693870067596436,2.1118621826171875,1.157524824142456
+,0.0002696193816368028,0.0002696193816368028,6349,,,,,,
+13,,,6349,2.533681869506836,1.3595455884933472,1.1741361618041992,,,
+,0.00041236376484242996,0.00041236376484242996,6399,,,,,,
+13,,,6399,1.7497609853744507,0.9464567303657532,0.8033042550086975,,,
+,7.806511533863421e-07,7.806511533863421e-07,6449,,,,,,
+13,,,6449,2.6392710208892822,0.9328999519348145,1.7063710689544678,,,
+,0.0004401026893699279,0.0004401026893699279,6499,,,,,,
+13,,,6499,2.2025022506713867,1.2049607038497925,0.9975415468215942,,,
+,0.00023039061836352398,0.00023039061836352398,6549,,,,,,
+13,,,6549,2.2280290126800537,1.1732120513916016,1.0548169612884521,,,
+,8.76462351576893e-05,8.76462351576893e-05,6599,,,,,,
+13,,,6599,1.7207846641540527,0.7194204926490784,1.0013641119003296,,,
+,0.0004992293488468644,0.0004992293488468644,6649,,,,,,
+13,,,6649,1.9033704996109009,0.9910165071487427,0.9123539924621582,,,
+,5.990731062987209e-05,5.990731062987209e-05,6699,,,,,,
+13,,,6699,2.0109341144561768,1.105412244796753,0.9055219292640686,,,
+,0.0002696193816367913,0.0002696193816367913,6749,,,,,,
+13,,,6749,1.999748945236206,0.9548845291137695,1.0448644161224365,,,
+,0.0004123637648429656,0.0004123637648429656,6799,,,,,,
+13,,,6799,1.9451771974563599,1.1922904253005981,0.7528867721557617,,,
+13,,,6831,,,,3.1334950923919678,1.9718761444091797,1.1616188287734985
+,7.806511533894499e-07,7.806511533894499e-07,6849,,,,,,
+14,,,6849,1.2186317443847656,0.5681224465370178,0.650509238243103,,,
+,0.00044010268937113195,0.00044010268937113195,6899,,,,,,
+14,,,6899,1.2228870391845703,0.6127429604530334,0.6101440191268921,,,
+,0.0002303906183634638,0.0002303906183634638,6949,,,,,,
+14,,,6949,1.8496158123016357,0.7061627507209778,1.1434531211853027,,,
+,8.764623515767546e-05,8.764623515767546e-05,6999,,,,,,
+14,,,6999,2.120867967605591,1.1101340055465698,1.010733962059021,,,
+,0.0004992293488467926,0.0004992293488467926,7049,,,,,,
+14,,,7049,1.787604570388794,0.9984426498413086,0.7891619205474854,,,
+,5.990731062988284e-05,5.990731062988284e-05,7099,,,,,,
+14,,,7099,2.9317374229431152,1.567914366722107,1.3638230562210083,,,
+,0.0002696193816367224,0.0002696193816367224,7149,,,,,,
+14,,,7149,1.7852485179901123,0.75360506772995,1.0316435098648071,,,
+,0.00041236376484289237,0.00041236376484289237,7199,,,,,,
+14,,,7199,2.643779754638672,1.569377064704895,1.0744028091430664,,,
+,7.806511533892007e-07,7.806511533892007e-07,7249,,,,,,
+14,,,7249,2.753891944885254,1.6068679094314575,1.147024154663086,,,
+,0.00044010268936978353,0.00044010268936978353,7299,,,,,,
+14,,,7299,1.4526660442352295,0.8541741967201233,0.5984917879104614,,,
+14,,,7319,,,,3.1202502250671387,1.96920907497406,1.1510415077209473
+,0.0002303906183634322,0.0002303906183634322,7349,,,,,,
+15,,,7349,1.5130116939544678,0.7608565092086792,0.7521551251411438,,,
+,8.76462351576834e-05,8.76462351576834e-05,7399,,,,,,
+15,,,7399,0.9867573976516724,0.46898144483566284,0.5177759528160095,,,
+,0.0004992293488467211,0.0004992293488467211,7449,,,,,,
+15,,,7449,1.733828067779541,0.710831344127655,1.0229967832565308,,,
+,5.99073106298567e-05,5.99073106298567e-05,7499,,,,,,
+15,,,7499,1.512115716934204,0.6120297312736511,0.9000859260559082,,,
+,0.0002696193816366821,0.0002696193816366821,7549,,,,,,
+15,,,7549,1.5944308042526245,0.8328548669815063,0.7615759372711182,,,
+,0.00041236376484283437,0.00041236376484283437,7599,,,,,,
+15,,,7599,1.542619228363037,0.8064069151878357,0.7362123727798462,,,
+,7.806511533867584e-07,7.806511533867584e-07,7649,,,,,,
+15,,,7649,1.8138139247894287,0.7319968342781067,1.0818171501159668,,,
+,0.0004401026893697195,0.0004401026893697195,7699,,,,,,
+15,,,7699,1.6971008777618408,0.8608031868934631,0.8362976312637329,,,
+,0.00023039061836406427,0.00023039061836406427,7749,,,,,,
+15,,,7749,1.2814245223999023,0.7309041619300842,0.5505204200744629,,,
+,8.764623515792211e-05,8.764623515792211e-05,7799,,,,,,
+15,,,7799,2.501283645629883,1.6320223808288574,0.8692613244056702,,,
+15,,,7807,,,,3.1231391429901123,1.9673525094985962,1.1557869911193848
+,0.0004992293488466491,0.0004992293488466491,7849,,,,,,
+16,,,7849,1.8004521131515503,0.8776185512542725,0.9228335618972778,,,
+,5.990731062984905e-05,5.990731062984905e-05,7899,,,,,,
+16,,,7899,1.7320818901062012,0.7657805681228638,0.9663012623786926,,,
+,0.00026961938163667026,0.00026961938163667026,7949,,,,,,
+16,,,7949,1.4932442903518677,0.5256585478782654,0.9675857424736023,,,
+,0.0004123637648427757,0.0004123637648427757,7999,,,,,,
+16,,,7999,1.9018189907073975,0.681605339050293,1.2202136516571045,,,
+,7.806511533887566e-07,7.806511533887566e-07,8049,,,,,,
+16,,,8049,1.4173989295959473,0.6298025846481323,0.7875964045524597,,,
+,0.000440102689370923,0.000440102689370923,8099,,,,,,
+16,,,8099,1.8658525943756104,0.9529937505722046,0.912858784198761,,,
+,0.00023039061836336043,0.00023039061836336043,8149,,,,,,
+16,,,8149,2.2028326988220215,1.374021053314209,0.828811526298523,,,
+,8.764623515763117e-05,8.764623515763117e-05,8199,,,,,,
+16,,,8199,1.9943840503692627,1.0718082189559937,0.922575831413269,,,
+,0.0004992293488465597,0.0004992293488465597,8249,,,,,,
+16,,,8249,1.7411184310913086,0.8104046583175659,0.9307138323783875,,,

checkpoint/no/AttR2Unet/wandb_logs/config.yaml

@@ -0,0 +1,157 @@
+# pytorch_lightning==2.5.1.post0
+seed_everything: 42
+trainer:
+  accelerator: cuda
+  strategy: ddp
+  devices:
+  - 2
+  num_nodes: 1
+  precision: 16-mixed
+  logger:
+  - class_path: pytorch_lightning.loggers.WandbLogger
+    init_args:
+      name: AttR2UnetNhaBe
+      save_dir: checkpoint/AttR2Unet/wandb_logs
+      version: null
+      offline: false
+      dir: null
+      id: null
+      anonymous: null
+      project: NhaBe
+      log_model: false
+      experiment: null
+      prefix: ''
+      checkpoint_name: null
+      entity: null
+      notes: null
+      tags: null
+      config: null
+      config_exclude_keys: null
+      config_include_keys: null
+      allow_val_change: null
+      group: null
+      job_type: null
+      mode: null
+      force: null
+      reinit: null
+      resume: null
+      resume_from: null
+      fork_from: null
+      save_code: null
+      tensorboard: null
+      sync_tensorboard: null
+      monitor_gym: null
+      settings: null
+  - class_path: pytorch_lightning.loggers.CSVLogger
+    init_args:
+      save_dir: checkpoint/AttR2Unet/csv_logs
+      name: null
+      version: null
+      prefix: ''
+      flush_logs_every_n_steps: 100
+  callbacks:
+  - class_path: pytorch_lightning.callbacks.LearningRateMonitor
+    init_args:
+      logging_interval: step
+      log_momentum: false
+      log_weight_decay: false
+  - class_path: pytorch_lightning.callbacks.ModelCheckpoint
+    init_args:
+      dirpath: checkpoint/AttR2Unet/checkpoints
+      filename: epoch_{epoch:03d}
+      monitor: val/mse
+      verbose: false
+      save_last: true
+      save_top_k: 1
+      save_weights_only: false
+      mode: min
+      auto_insert_metric_name: false
+      every_n_train_steps: null
+      train_time_interval: null
+      every_n_epochs: null
+      save_on_train_epoch_end: null
+      enable_version_counter: true
+  - class_path: pytorch_lightning.callbacks.EarlyStopping
+    init_args:
+      monitor: val/mse
+      min_delta: 0.0
+      patience: 10
+      verbose: false
+      mode: min
+      strict: true
+      check_finite: true
+      stopping_threshold: null
+      divergence_threshold: null
+      check_on_train_epoch_end: null
+      log_rank_zero_only: false
+  - class_path: pytorch_lightning.callbacks.RichModelSummary
+    init_args:
+      max_depth: -1
+  fast_dev_run: false
+  max_epochs: 100
+  min_epochs: 1
+  max_steps: -1
+  min_steps: null
+  max_time: null
+  limit_train_batches: null
+  limit_val_batches: null
+  limit_test_batches: null
+  limit_predict_batches: null
+  overfit_batches: 0.0
+  val_check_interval: null
+  check_val_every_n_epoch: 1
+  num_sanity_val_steps: null
+  log_every_n_steps: null
+  enable_checkpointing: true
+  enable_progress_bar: true
+  enable_model_summary: null
+  accumulate_grad_batches: 1
+  gradient_clip_val: null
+  gradient_clip_algorithm: null
+  deterministic: null
+  benchmark: null
+  inference_mode: true
+  use_distributed_sampler: true
+  profiler: null
+  detect_anomaly: false
+  barebones: false
+  plugins: null
+  sync_batchnorm: true
+  reload_dataloaders_every_n_epochs: 0
+  default_root_dir: checkpoint/AttR2Unet
+  model_registry: null
+model:
+  net:
+    class_path: arch.Network
+    init_args:
+      model_type: AttR2Unet
+      rad_channel: 1
+      sat_channel: 1
+      rad_size: 400
+      sat_size: 25
+  pretrained_path: ''
+  lr: 0.0005
+  beta_1: 0.9
+  beta_2: 0.99
+  weight_decay: 1.0e-05
+  warmup_epochs: 10
+  max_epochs: 50
+  warmup_start_lr: 1.0e-08
+  eta_min: 1.0e-08
+data:
+  dir_data: /cm/archive/vinhbk1/NhaBe
+  batch_size: 64
+  hours_predicted: 3
+  num_workers: 8
+  pin_memory: false
+  time_points_rad: 1
+  time_points_sat: 1
+  sat_inp_vars: total_precipitation
+  sat_out_vars: total_precipitation
+  sat_size: 25
+  rad_inp_vars: precipitation
+  rad_out_vars: precipitation
+  rad_size: 400
+  ablation: 'no'
+optimizer: null
+lr_scheduler: null

checkpoint/no/AttR2Unet/wandb_logs/wandb/debug-internal.log

@@ -0,0 +1,7 @@
+{"time":"2025-06-30T03:58:45.70544069+07:00","level":"INFO","msg":"stream: starting","core version":"0.20.1","symlink path":"checkpoint/AttR2Unet/wandb_logs/wandb/run-20250630_035845-4hs9k1ow/logs/debug-core.log"}
+{"time":"2025-06-30T03:58:46.19712103+07:00","level":"INFO","msg":"stream: created new stream","id":"4hs9k1ow"}
+{"time":"2025-06-30T03:58:46.19715788+07:00","level":"INFO","msg":"stream: started","id":"4hs9k1ow"}
+{"time":"2025-06-30T03:58:46.197170754+07:00","level":"INFO","msg":"handler: started","stream_id":"4hs9k1ow"}
+{"time":"2025-06-30T03:58:46.197175954+07:00","level":"INFO","msg":"sender: started","stream_id":"4hs9k1ow"}
+{"time":"2025-06-30T03:58:46.197189139+07:00","level":"INFO","msg":"writer: Do: started","stream_id":"4hs9k1ow"}
+{"time":"2025-06-30T03:58:46.524582874+07:00","level":"INFO","msg":"Starting system monitor"}

checkpoint/no/AttR2Unet/wandb_logs/wandb/debug.log

@@ -0,0 +1,22 @@
+2025-06-30 03:58:45,471 INFO    MainThread:3084621 [wandb_setup.py:_flush():81] Current SDK version is 0.20.1
+2025-06-30 03:58:45,471 INFO    MainThread:3084621 [wandb_setup.py:_flush():81] Configure stats pid to 3084621
+2025-06-30 03:58:45,471 INFO    MainThread:3084621 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/.config/wandb/settings
+2025-06-30 03:58:45,473 INFO    MainThread:3084621 [wandb_setup.py:_flush():81] Loading settings from /home/vinhbk1/weatherforecast/unet/wandb/settings
+2025-06-30 03:58:45,473 INFO    MainThread:3084621 [wandb_setup.py:_flush():81] Loading settings from environment variables
+2025-06-30 03:58:45,473 INFO    MainThread:3084621 [wandb_init.py:setup_run_log_directory():703] Logging user logs to checkpoint/AttR2Unet/wandb_logs/wandb/run-20250630_035845-4hs9k1ow/logs/debug.log
+2025-06-30 03:58:45,474 INFO    MainThread:3084621 [wandb_init.py:setup_run_log_directory():704] Logging internal logs to checkpoint/AttR2Unet/wandb_logs/wandb/run-20250630_035845-4hs9k1ow/logs/debug-internal.log
+2025-06-30 03:58:45,474 INFO    MainThread:3084621 [wandb_init.py:init():831] calling init triggers
+2025-06-30 03:58:45,475 INFO    MainThread:3084621 [wandb_init.py:init():836] wandb.init called with sweep_config: {}
+config: {'_wandb': {}}
+2025-06-30 03:58:45,475 INFO    MainThread:3084621 [wandb_init.py:init():872] starting backend
+2025-06-30 03:58:45,692 INFO    MainThread:3084621 [wandb_init.py:init():875] sending inform_init request
+2025-06-30 03:58:45,697 INFO    MainThread:3084621 [wandb_init.py:init():883] backend started and connected
+2025-06-30 03:58:45,698 INFO    MainThread:3084621 [wandb_init.py:init():956] updated telemetry
+2025-06-30 03:58:45,700 INFO    MainThread:3084621 [wandb_init.py:init():980] communicating run to backend with 90.0 second timeout
+2025-06-30 03:58:46,513 INFO    MainThread:3084621 [wandb_init.py:init():1032] starting run threads in backend
+2025-06-30 03:58:46,636 INFO    MainThread:3084621 [wandb_run.py:_console_start():2453] atexit reg
+2025-06-30 03:58:46,636 INFO    MainThread:3084621 [wandb_run.py:_redirect():2301] redirect: wrap_raw
+2025-06-30 03:58:46,636 INFO    MainThread:3084621 [wandb_run.py:_redirect():2370] Wrapping output streams.
+2025-06-30 03:58:46,638 INFO    MainThread:3084621 [wandb_run.py:_redirect():2393] Redirects installed.
+2025-06-30 03:58:46,643 INFO    MainThread:3084621 [wandb_init.py:init():1078] run started, returning control to user process
+2025-06-30 03:58:53,295 INFO    MainThread:3084621 [wandb_run.py:_config_callback():1358] config_cb None None {'pretrained_path': '', 'lr': 0.0005, 'beta_1': 0.9, 'beta_2': 0.99, 'weight_decay': 1e-05, 'warmup_epochs': 10, 'max_epochs': 50, 'warmup_start_lr': 1e-08, 'eta_min': 1e-08, '_instantiator': 'pytorch_lightning.cli.instantiate_module', 'dir_data': '/cm/archive/vinhbk1/NhaBe', 'batch_size': 64, 'hours_predicted': 3, 'num_workers': 8, 'pin_memory': False, 'time_points_rad': 1, 'time_points_sat': 1, 'sat_inp_vars': 'total_precipitation', 'sat_out_vars': 'total_precipitation', 'sat_size': 25, 'rad_inp_vars': 'precipitation', 'rad_out_vars': 'precipitation', 'rad_size': 400, 'ablation': 'no'}

checkpoint/no/AttR2Unet/wandb_logs/wandb/run-20250630_035845-4hs9k1ow/files/output.log

@@ -0,0 +1,250 @@
+Number of train samples: 31187
+Number of test samples: 8077
+Number of val samples: 1398
+LOCAL_RANK: 0 - CUDA_VISIBLE_DEVICES: [0,1,2,3,4,5,6,7]
+┏━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┓
+┃     ┃ Name                                   ┃ Type        ┃ Params ┃ Mode  ┃
+┡━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━╇━━━━━━━━╇━━━━━━━┩
+│ 0   │ net                                    │ Network     │ 28.1 K │ train │
+│ 1   │ net.net                                │ AttR2Unet   │ 28.1 K │ train │
+│ 2   │ net.net.encoder_blocks                 │ ModuleList  │  6.5 K │ train │
+│ 3   │ net.net.encoder_blocks.0               │ RRCNN_block │     88 │ train │
+│ 4   │ net.net.encoder_blocks.0.RCNN          │ Sequential  │     84 │ train │
+│ 5   │ net.net.encoder_blocks.0.RCNN.0        │ Recurrent_… │     42 │ train │
+│ 6   │ net.net.encoder_blocks.0.RCNN.0.conv   │ Sequential  │     42 │ train │
+│ 7   │ net.net.encoder_blocks.0.RCNN.0.conv.0 │ Conv2d      │     38 │ train │
+│ 8   │ net.net.encoder_blocks.0.RCNN.0.conv.1 │ SyncBatchN… │      4 │ train │
+│ 9   │ net.net.encoder_blocks.0.RCNN.0.conv.2 │ ReLU        │      0 │ train │
+│ 10  │ net.net.encoder_blocks.0.RCNN.1        │ Recurrent_… │     42 │ train │
+│ 11  │ net.net.encoder_blocks.0.RCNN.1.conv   │ Sequential  │     42 │ train │
+│ 12  │ net.net.encoder_blocks.0.RCNN.1.conv.0 │ Conv2d      │     38 │ train │
+│ 13  │ net.net.encoder_blocks.0.RCNN.1.conv.1 │ SyncBatchN… │      4 │ train │
+│ 14  │ net.net.encoder_blocks.0.RCNN.1.conv.2 │ ReLU        │      0 │ train │
+│ 15  │ net.net.encoder_blocks.0.Conv_1x1      │ Conv2d      │      4 │ train │
+│ 16  │ net.net.encoder_blocks.1               │ RRCNN_block │    324 │ train │
+│ 17  │ net.net.encoder_blocks.1.RCNN          │ Sequential  │    312 │ train │
+│ 18  │ net.net.encoder_blocks.1.RCNN.0        │ Recurrent_… │    156 │ train │
+│ 19  │ net.net.encoder_blocks.1.RCNN.0.conv   │ Sequential  │    156 │ train │
+│ 20  │ net.net.encoder_blocks.1.RCNN.0.conv.0 │ Conv2d      │    148 │ train │
+│ 21  │ net.net.encoder_blocks.1.RCNN.0.conv.1 │ SyncBatchN… │      8 │ train │
+│ 22  │ net.net.encoder_blocks.1.RCNN.0.conv.2 │ ReLU        │      0 │ train │
+│ 23  │ net.net.encoder_blocks.1.RCNN.1        │ Recurrent_… │    156 │ train │
+│ 24  │ net.net.encoder_blocks.1.RCNN.1.conv   │ Sequential  │    156 │ train │
+│ 25  │ net.net.encoder_blocks.1.RCNN.1.conv.0 │ Conv2d      │    148 │ train │
+│ 26  │ net.net.encoder_blocks.1.RCNN.1.conv.1 │ SyncBatchN… │      8 │ train │
+│ 27  │ net.net.encoder_blocks.1.RCNN.1.conv.2 │ ReLU        │      0 │ train │
+│ 28  │ net.net.encoder_blocks.1.Conv_1x1      │ Conv2d      │     12 │ train │
+│ 29  │ net.net.encoder_blocks.2               │ RRCNN_block │  1.2 K │ train │
+│ 30  │ net.net.encoder_blocks.2.RCNN          │ Sequential  │  1.2 K │ train │
+│ 31  │ net.net.encoder_blocks.2.RCNN.0        │ Recurrent_… │    600 │ train │
+│ 32  │ net.net.encoder_blocks.2.RCNN.0.conv   │ Sequential  │    600 │ train │
+│ 33  │ net.net.encoder_blocks.2.RCNN.0.conv.0 │ Conv2d      │    584 │ train │
+│ 34  │ net.net.encoder_blocks.2.RCNN.0.conv.1 │ SyncBatchN… │     16 │ train │
+│ 35  │ net.net.encoder_blocks.2.RCNN.0.conv.2 │ ReLU        │      0 │ train │
+│ 36  │ net.net.encoder_blocks.2.RCNN.1        │ Recurrent_… │    600 │ train │
+│ 37  │ net.net.encoder_blocks.2.RCNN.1.conv   │ Sequential  │    600 │ train │
+│ 38  │ net.net.encoder_blocks.2.RCNN.1.conv.0 │ Conv2d      │    584 │ train │
+│ 39  │ net.net.encoder_blocks.2.RCNN.1.conv.1 │ SyncBatchN… │     16 │ train │
+│ 40  │ net.net.encoder_blocks.2.RCNN.1.conv.2 │ ReLU        │      0 │ train │
+│ 41  │ net.net.encoder_blocks.2.Conv_1x1      │ Conv2d      │     40 │ train │
+│ 42  │ net.net.encoder_blocks.3               │ RRCNN_block │  4.8 K │ train │
+│ 43  │ net.net.encoder_blocks.3.RCNN          │ Sequential  │  4.7 K │ train │
+│ 44  │ net.net.encoder_blocks.3.RCNN.0        │ Recurrent_… │  2.4 K │ train │
+│ 45  │ net.net.encoder_blocks.3.RCNN.0.conv   │ Sequential  │  2.4 K │ train │
+│ 46  │ net.net.encoder_blocks.3.RCNN.0.conv.0 │ Conv2d      │  2.3 K │ train │
+│ 47  │ net.net.encoder_blocks.3.RCNN.0.conv.1 │ SyncBatchN… │     32 │ train │
+│ 48  │ net.net.encoder_blocks.3.RCNN.0.conv.2 │ ReLU        │      0 │ train │
+│ 49  │ net.net.encoder_blocks.3.RCNN.1        │ Recurrent_… │  2.4 K │ train │
+│ 50  │ net.net.encoder_blocks.3.RCNN.1.conv   │ Sequential  │  2.4 K │ train │
+│ 51  │ net.net.encoder_blocks.3.RCNN.1.conv.0 │ Conv2d      │  2.3 K │ train │
+│ 52  │ net.net.encoder_blocks.3.RCNN.1.conv.1 │ SyncBatchN… │     32 │ train │
+│ 53  │ net.net.encoder_blocks.3.RCNN.1.conv.2 │ ReLU        │      0 │ train │
+│ 54  │ net.net.encoder_blocks.3.Conv_1x1      │ Conv2d      │    144 │ train │
+│ 55  │ net.net.pools                          │ ModuleList  │      0 │ train │
+│ 56  │ net.net.pools.0                        │ MaxPool2d   │      0 │ train │
+│ 57  │ net.net.pools.1                        │ MaxPool2d   │      0 │ train │
+│ 58  │ net.net.pools.2                        │ MaxPool2d   │      0 │ train │
+│ 59  │ net.net.pools.3                        │ MaxPool2d   │      0 │ train │
+│ 60  │ net.net.mid_conv_1                     │ single_conv │  2.4 K │ train │
+│ 61  │ net.net.mid_conv_1.conv                │ Sequential  │  2.4 K │ train │
+│ 62  │ net.net.mid_conv_1.conv.0              │ Conv2d      │  2.3 K │ train │
+│ 63  │ net.net.mid_conv_1.conv.1              │ SyncBatchN… │     32 │ train │
+│ 64  │ net.net.mid_conv_1.conv.2              │ ReLU        │      0 │ train │
+│ 65  │ net.net.mid_conv_2                     │ single_conv │    192 │ train │
+│ 66  │ net.net.mid_conv_2.conv                │ Sequential  │    192 │ train │
+│ 67  │ net.net.mid_conv_2.conv.0              │ Conv2d      │    160 │ train │
+│ 68  │ net.net.mid_conv_2.conv.1              │ SyncBatchN… │     32 │ train │
+│ 69  │ net.net.mid_conv_2.conv.2              │ ReLU        │      0 │ train │
+│ 70  │ net.net.mid_merge                      │ RRCNN_block │  5.2 K │ train │
+│ 71  │ net.net.mid_merge.RCNN                 │ Sequential  │  4.7 K │ train │
+│ 72  │ net.net.mid_merge.RCNN.0               │ Recurrent_… │  2.4 K │ train │
+│ 73  │ net.net.mid_merge.RCNN.0.conv          │ Sequential  │  2.4 K │ train │
+│ 74  │ net.net.mid_merge.RCNN.0.conv.0        │ Conv2d      │  2.3 K │ train │
+│ 75  │ net.net.mid_merge.RCNN.0.conv.1        │ SyncBatchN… │     32 │ train │
+│ 76  │ net.net.mid_merge.RCNN.0.conv.2        │ ReLU        │      0 │ train │
+│ 77  │ net.net.mid_merge.RCNN.1               │ Recurrent_… │  2.4 K │ train │
+│ 78  │ net.net.mid_merge.RCNN.1.conv          │ Sequential  │  2.4 K │ train │
+│ 79  │ net.net.mid_merge.RCNN.1.conv.0        │ Conv2d      │  2.3 K │ train │
+│ 80  │ net.net.mid_merge.RCNN.1.conv.1        │ SyncBatchN… │     32 │ train │
+│ 81  │ net.net.mid_merge.RCNN.1.conv.2        │ ReLU        │      0 │ train │
+│ 82  │ net.net.mid_merge.Conv_1x1             │ Conv2d      │    528 │ train │
+│ 83  │ net.net.up_convs                       │ ModuleList  │  6.2 K │ train │
+│ 84  │ net.net.up_convs.0                     │ UpConv      │  4.7 K │ train │
+│ 85  │ net.net.up_convs.0.up                  │ Sequential  │  4.7 K │ train │
+│ 86  │ net.net.up_convs.0.up.0                │ Upsample    │      0 │ train │
+│ 87  │ net.net.up_convs.0.up.1                │ Conv2d      │  4.6 K │ train │
+│ 88  │ net.net.up_convs.0.up.2                │ SyncBatchN… │     32 │ train │
+│ 89  │ net.net.up_convs.0.up.3                │ ReLU        │      0 │ train │
+│ 90  │ net.net.up_convs.1                     │ UpConv      │  1.2 K │ train │
+│ 91  │ net.net.up_convs.1.up                  │ Sequential  │  1.2 K │ train │
+│ 92  │ net.net.up_convs.1.up.0                │ Upsample    │      0 │ train │
+│ 93  │ net.net.up_convs.1.up.1                │ Conv2d      │  1.2 K │ train │
+│ 94  │ net.net.up_convs.1.up.2                │ SyncBatchN… │     16 │ train │
+│ 95  │ net.net.up_convs.1.up.3                │ ReLU        │      0 │ train │
+│ 96  │ net.net.up_convs.2                     │ UpConv      │    300 │ train │
+│ 97  │ net.net.up_convs.2.up                  │ Sequential  │    300 │ train │
+│ 98  │ net.net.up_convs.2.up.0                │ Upsample    │      0 │ train │
+│ 99  │ net.net.up_convs.2.up.1                │ Conv2d      │    292 │ train │
+│ 100 │ net.net.up_convs.2.up.2                │ SyncBatchN… │      8 │ train │
+│ 101 │ net.net.up_convs.2.up.3                │ ReLU        │      0 │ train │
+│ 102 │ net.net.up_convs.3                     │ UpConv      │     78 │ train │
+│ 103 │ net.net.up_convs.3.up                  │ Sequential  │     78 │ train │
+│ 104 │ net.net.up_convs.3.up.0                │ Upsample    │      0 │ train │
+│ 105 │ net.net.up_convs.3.up.1                │ Conv2d      │     74 │ train │
+│ 106 │ net.net.up_convs.3.up.2                │ SyncBatchN… │      4 │ train │
+│ 107 │ net.net.up_convs.3.up.3                │ ReLU        │      0 │ train │
+│ 108 │ net.net.attention_blocks               │ ModuleList  │    457 │ train │
+│ 109 │ net.net.attention_blocks.0             │ AttentionB… │    315 │ train │
+│ 110 │ net.net.attention_blocks.0.W_gate      │ Sequential  │    152 │ train │
+│ 111 │ net.net.attention_blocks.0.W_gate.0    │ Conv2d      │    136 │ train │
+│ 112 │ net.net.attention_blocks.0.W_gate.1    │ SyncBatchN… │     16 │ train │
+│ 113 │ net.net.attention_blocks.0.W_x         │ Sequential  │    152 │ train │
+│ 114 │ net.net.attention_blocks.0.W_x.0       │ Conv2d      │    136 │ train │
+│ 115 │ net.net.attention_blocks.0.W_x.1       │ SyncBatchN… │     16 │ train │
+│ 116 │ net.net.attention_blocks.0.psi         │ Sequential  │     11 │ train │
+│ 117 │ net.net.attention_blocks.0.psi.0       │ Conv2d      │      9 │ train │
+│ 118 │ net.net.attention_blocks.0.psi.1       │ SyncBatchN… │      2 │ train │
+│ 119 │ net.net.attention_blocks.0.psi.2       │ Sigmoid     │      0 │ train │
+│ 120 │ net.net.attention_blocks.0.relu        │ ReLU        │      0 │ train │
+│ 121 │ net.net.attention_blocks.1             │ AttentionB… │     95 │ train │
+│ 122 │ net.net.attention_blocks.1.W_gate      │ Sequential  │     44 │ train │
+│ 123 │ net.net.attention_blocks.1.W_gate.0    │ Conv2d      │     36 │ train │
+│ 124 │ net.net.attention_blocks.1.W_gate.1    │ SyncBatchN… │      8 │ train │
+│ 125 │ net.net.attention_blocks.1.W_x         │ Sequential  │     44 │ train │
+│ 126 │ net.net.attention_blocks.1.W_x.0       │ Conv2d      │     36 │ train │
+│ 127 │ net.net.attention_blocks.1.W_x.1       │ SyncBatchN… │      8 │ train │
+│ 128 │ net.net.attention_blocks.1.psi         │ Sequential  │      7 │ train │
+│ 129 │ net.net.attention_blocks.1.psi.0       │ Conv2d      │      5 │ train │
+│ 130 │ net.net.attention_blocks.1.psi.1       │ SyncBatchN… │      2 │ train │
+│ 131 │ net.net.attention_blocks.1.psi.2       │ Sigmoid     │      0 │ train │
+│ 132 │ net.net.attention_blocks.1.relu        │ ReLU        │      0 │ train │
+│ 133 │ net.net.attention_blocks.2             │ AttentionB… │     33 │ train │
+│ 134 │ net.net.attention_blocks.2.W_gate      │ Sequential  │     14 │ train │
+│ 135 │ net.net.attention_blocks.2.W_gate.0    │ Conv2d      │     10 │ train │
+│ 136 │ net.net.attention_blocks.2.W_gate.1    │ SyncBatchN… │      4 │ train │
+│ 137 │ net.net.attention_blocks.2.W_x         │ Sequential  │     14 │ train │
+│ 138 │ net.net.attention_blocks.2.W_x.0       │ Conv2d      │     10 │ train │
+│ 139 │ net.net.attention_blocks.2.W_x.1       │ SyncBatchN… │      4 │ train │
+│ 140 │ net.net.attention_blocks.2.psi         │ Sequential  │      5 │ train │
+│ 141 │ net.net.attention_blocks.2.psi.0       │ Conv2d      │      3 │ train │
+│ 142 │ net.net.attention_blocks.2.psi.1       │ SyncBatchN… │      2 │ train │
+│ 143 │ net.net.attention_blocks.2.psi.2       │ Sigmoid     │      0 │ train │
+│ 144 │ net.net.attention_blocks.2.relu        │ ReLU        │      0 │ train │
+│ 145 │ net.net.attention_blocks.3             │ AttentionB… │     14 │ train │
+│ 146 │ net.net.attention_blocks.3.W_gate      │ Sequential  │      5 │ train │
+│ 147 │ net.net.attention_blocks.3.W_gate.0    │ Conv2d      │      3 │ train │
+│ 148 │ net.net.attention_blocks.3.W_gate.1    │ SyncBatchN… │      2 │ train │
+│ 149 │ net.net.attention_blocks.3.W_x         │ Sequential  │      5 │ train │
+│ 150 │ net.net.attention_blocks.3.W_x.0       │ Conv2d      │      3 │ train │
+│ 151 │ net.net.attention_blocks.3.W_x.1       │ SyncBatchN… │      2 │ train │
+│ 152 │ net.net.attention_blocks.3.psi         │ Sequential  │      4 │ train │
+│ 153 │ net.net.attention_blocks.3.psi.0       │ Conv2d      │      2 │ train │
+│ 154 │ net.net.attention_blocks.3.psi.1       │ SyncBatchN… │      2 │ train │
+│ 155 │ net.net.attention_blocks.3.psi.2       │ Sigmoid     │      0 │ train │
+│ 156 │ net.net.attention_blocks.3.relu        │ ReLU        │      0 │ train │
+│ 157 │ net.net.decoder_blocks                 │ ModuleList  │  7.0 K │ train │
+│ 158 │ net.net.decoder_blocks.0               │ RRCNN_block │  5.2 K │ train │
+│ 159 │ net.net.decoder_blocks.0.RCNN          │ Sequential  │  4.7 K │ train │
+│ 160 │ net.net.decoder_blocks.0.RCNN.0        │ Recurrent_… │  2.4 K │ train │
+│ 161 │ net.net.decoder_blocks.0.RCNN.0.conv   │ Sequential  │  2.4 K │ train │
+│ 162 │ net.net.decoder_blocks.0.RCNN.0.conv.0 │ Conv2d      │  2.3 K │ train │
+│ 163 │ net.net.decoder_blocks.0.RCNN.0.conv.1 │ SyncBatchN… │     32 │ train │
+│ 164 │ net.net.decoder_blocks.0.RCNN.0.conv.2 │ ReLU        │      0 │ train │
+│ 165 │ net.net.decoder_blocks.0.RCNN.1        │ Recurrent_… │  2.4 K │ train │
+│ 166 │ net.net.decoder_blocks.0.RCNN.1.conv   │ Sequential  │  2.4 K │ train │
+│ 167 │ net.net.decoder_blocks.0.RCNN.1.conv.0 │ Conv2d      │  2.3 K │ train │
+│ 168 │ net.net.decoder_blocks.0.RCNN.1.conv.1 │ SyncBatchN… │     32 │ train │
+│ 169 │ net.net.decoder_blocks.0.RCNN.1.conv.2 │ ReLU        │      0 │ train │
+│ 170 │ net.net.decoder_blocks.0.Conv_1x1      │ Conv2d      │    528 │ train │
+│ 171 │ net.net.decoder_blocks.1               │ RRCNN_block │  1.3 K │ train │
+│ 172 │ net.net.decoder_blocks.1.RCNN          │ Sequential  │  1.2 K │ train │
+│ 173 │ net.net.decoder_blocks.1.RCNN.0        │ Recurrent_… │    600 │ train │
+│ 174 │ net.net.decoder_blocks.1.RCNN.0.conv   │ Sequential  │    600 │ train │
+│ 175 │ net.net.decoder_blocks.1.RCNN.0.conv.0 │ Conv2d      │    584 │ train │
+│ 176 │ net.net.decoder_blocks.1.RCNN.0.conv.1 │ SyncBatchN… │     16 │ train │
+│ 177 │ net.net.decoder_blocks.1.RCNN.0.conv.2 │ ReLU        │      0 │ train │
+│ 178 │ net.net.decoder_blocks.1.RCNN.1        │ Recurrent_… │    600 │ train │
+│ 179 │ net.net.decoder_blocks.1.RCNN.1.conv   │ Sequential  │    600 │ train │
+│ 180 │ net.net.decoder_blocks.1.RCNN.1.conv.0 │ Conv2d      │    584 │ train │
+│ 181 │ net.net.decoder_blocks.1.RCNN.1.conv.1 │ SyncBatchN… │     16 │ train │
+│ 182 │ net.net.decoder_blocks.1.RCNN.1.conv.2 │ ReLU        │      0 │ train │
+│ 183 │ net.net.decoder_blocks.1.Conv_1x1      │ Conv2d      │    136 │ train │
+│ 184 │ net.net.decoder_blocks.2               │ RRCNN_block │    348 │ train │
+│ 185 │ net.net.decoder_blocks.2.RCNN          │ Sequential  │    312 │ train │
+│ 186 │ net.net.decoder_blocks.2.RCNN.0        │ Recurrent_… │    156 │ train │
+│ 187 │ net.net.decoder_blocks.2.RCNN.0.conv   │ Sequential  │    156 │ train │
+│ 188 │ net.net.decoder_blocks.2.RCNN.0.conv.0 │ Conv2d      │    148 │ train │
+│ 189 │ net.net.decoder_blocks.2.RCNN.0.conv.1 │ SyncBatchN… │      8 │ train │
+│ 190 │ net.net.decoder_blocks.2.RCNN.0.conv.2 │ ReLU        │      0 │ train │
+│ 191 │ net.net.decoder_blocks.2.RCNN.1        │ Recurrent_… │    156 │ train │
+│ 192 │ net.net.decoder_blocks.2.RCNN.1.conv   │ Sequential  │    156 │ train │
+│ 193 │ net.net.decoder_blocks.2.RCNN.1.conv.0 │ Conv2d      │    148 │ train │
+│ 194 │ net.net.decoder_blocks.2.RCNN.1.conv.1 │ SyncBatchN… │      8 │ train │
+│ 195 │ net.net.decoder_blocks.2.RCNN.1.conv.2 │ ReLU        │      0 │ train │
+│ 196 │ net.net.decoder_blocks.2.Conv_1x1      │ Conv2d      │     36 │ train │
+│ 197 │ net.net.decoder_blocks.3               │ RRCNN_block │     94 │ train │
+│ 198 │ net.net.decoder_blocks.3.RCNN          │ Sequential  │     84 │ train │
+│ 199 │ net.net.decoder_blocks.3.RCNN.0        │ Recurrent_… │     42 │ train │
+│ 200 │ net.net.decoder_blocks.3.RCNN.0.conv   │ Sequential  │     42 │ train │
+│ 201 │ net.net.decoder_blocks.3.RCNN.0.conv.0 │ Conv2d      │     38 │ train │
+│ 202 │ net.net.decoder_blocks.3.RCNN.0.conv.1 │ SyncBatchN… │      4 │ train │
+│ 203 │ net.net.decoder_blocks.3.RCNN.0.conv.2 │ ReLU        │      0 │ train │
+│ 204 │ net.net.decoder_blocks.3.RCNN.1        │ Recurrent_… │     42 │ train │
+│ 205 │ net.net.decoder_blocks.3.RCNN.1.conv   │ Sequential  │     42 │ train │
+│ 206 │ net.net.decoder_blocks.3.RCNN.1.conv.0 │ Conv2d      │     38 │ train │
+│ 207 │ net.net.decoder_blocks.3.RCNN.1.conv.1 │ SyncBatchN… │      4 │ train │
+│ 208 │ net.net.decoder_blocks.3.RCNN.1.conv.2 │ ReLU        │      0 │ train │
+│ 209 │ net.net.decoder_blocks.3.Conv_1x1      │ Conv2d      │     10 │ train │
+│ 210 │ net.net.final_decoder                  │ RRCNN_block │     94 │ train │
+│ 211 │ net.net.final_decoder.RCNN             │ Sequential  │     84 │ train │
+│ 212 │ net.net.final_decoder.RCNN.0           │ Recurrent_… │     42 │ train │
+│ 213 │ net.net.final_decoder.RCNN.0.conv      │ Sequential  │     42 │ train │
+│ 214 │ net.net.final_decoder.RCNN.0.conv.0    │ Conv2d      │     38 │ train │
+│ 215 │ net.net.final_decoder.RCNN.0.conv.1    │ SyncBatchN… │      4 │ train │
+│ 216 │ net.net.final_decoder.RCNN.0.conv.2    │ ReLU        │      0 │ train │
+│ 217 │ net.net.final_decoder.RCNN.1           │ Recurrent_… │     42 │ train │
+│ 218 │ net.net.final_decoder.RCNN.1.conv      │ Sequential  │     42 │ train │
+│ 219 │ net.net.final_decoder.RCNN.1.conv.0    │ Conv2d      │     38 │ train │
+│ 220 │ net.net.final_decoder.RCNN.1.conv.1    │ SyncBatchN… │      4 │ train │
+│ 221 │ net.net.final_decoder.RCNN.1.conv.2    │ ReLU        │      0 │ train │
+│ 222 │ net.net.final_decoder.Conv_1x1         │ Conv2d      │     10 │ train │
+│ 223 │ net.net.out_conv_R                     │ Conv2d      │      3 │ train │
+│ 224 │ net.net.out_conv_S                     │ Conv2d      │     17 │ train │
+│ 225 │ rad_denormalization                    │ Normalize   │      0 │ train │
+│ 226 │ sat_denormalization                    │ Normalize   │      0 │ train │
+└─────┴────────────────────────────────────────┴─────────────┴────────┴───────┘
+Trainable params: 28.1 K
+Non-trainable params: 0
+Total params: 28.1 K
+Total estimated model params size (MB): 0
+Modules in train mode: 227
+Modules in eval mode: 0
+Epoch 16:  98%|██████████████████████████████▍| 480/488 [20:29<00:20,  0.39it/s, v_num=ow_0, train/rad=0.594, train/sat=1.000, train/mse=1.590, val/rad=1.970, val/sat=1.160, val/mse=3.120]
+/cm/shared/miniconda3/envs/unet/lib/python3.10/site-packages/torch/distributed/distributed_c10d.py:4631: UserWarning: No device id is provided via `init_process_group` or `barrier `. Using the current device set by the user.
+  warnings.warn(  # warn only once
+/cm/shared/miniconda3/envs/unet/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('val/rad', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/cm/shared/miniconda3/envs/unet/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('val/sat', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/cm/shared/miniconda3/envs/unet/lib/python3.10/site-packages/pytorch_lightning/trainer/connectors/logger_connector/result.py:434: It is recommended to use `self.log('val/mse', ..., sync_dist=True)` when logging on epoch level in distributed setting to accumulate the metric across devices.
+/cm/shared/miniconda3/envs/unet/lib/python3.10/site-packages/torch/optim/lr_scheduler.py:182: UserWarning: Detected call of `lr_scheduler.step()` before `optimizer.step()`. In PyTorch 1.1.0 and later, you should call them in the opposite order: `optimizer.step()` before `lr_scheduler.step()`.  Failure to do this will result in PyTorch skipping the first value of the learning rate schedule. See more details at https://pytorch.org/docs/stable/optim.html#how-to-adjust-learning-rate
+  warnings.warn(
+                                                                               

checkpoint/no/AttR2Unet/wandb_logs/wandb/run-20250630_035845-4hs9k1ow/files/requirements.txt

@@ -0,0 +1,84 @@
+async-timeout==5.0.1
+annotated-types==0.7.0
+certifi==2025.6.15
+typing_extensions==4.14.0
+nvidia-cuda-runtime-cu12==12.6.77
+packaging==25.0
+pydantic_core==2.33.2
+triton==3.3.1
+sympy==1.14.0
+nvidia-cufft-cu12==11.3.0.4
+charset-normalizer==3.4.2
+nvidia-cusparselt-cu12==0.6.3
+nvidia-cublas-cu12==12.6.4.1
+fsspec==2025.5.1
+pytorch-lightning==2.5.1.post0
+hf-xet==1.1.5
+attrs==25.3.0
+pydantic==2.11.6
+webencodings==0.5.1
+nvidia-cusolver-cu12==11.7.1.2
+aiohappyeyeballs==2.6.1
+python-dateutil==2.9.0.post0
+numpy==2.2.6
+torchsummary==1.5.1
+pip==25.1
+idna==3.10
+nvidia-cusparse-cu12==12.5.4.2
+nvidia-cuda-nvrtc-cu12==12.6.77
+setproctitle==1.3.6
+kaggle==1.7.4.5
+Jinja2==3.1.6
+MarkupSafe==3.0.2
+nvidia-nvtx-cu12==12.6.77
+climax==0.3.1
+wheel==0.45.1
+requests==2.32.4
+urllib3==2.4.0
+yarl==1.20.1
+rich==14.0.0
+six==1.17.0
+nvidia-cuda-cupti-cu12==12.6.80
+mpmath==1.3.0
+nvidia-nvjitlink-cu12==12.6.85
+bleach==6.2.0
+torchvision==0.22.1
+gitdb==4.0.12
+aiosignal==1.3.2
+psutil==7.0.0
+multidict==6.4.4
+pillow==11.2.1
+text-unidecode==1.3
+PyYAML==6.0.2
+typing-inspection==0.4.1
+lightning-utilities==0.14.3
+protobuf==6.31.1
+sentry-sdk==2.30.0
+typeshed_client==2.7.0
+smmap==5.0.2
+torchaudio==2.7.1
+tqdm==4.67.1
+wandb==0.20.1
+docstring_parser==0.16
+nvidia-cufile-cu12==1.11.1.6
+aiohttp==3.12.12
+nvidia-nccl-cu12==2.26.2
+mdurl==0.1.2
+huggingface-hub==0.33.0
+filelock==3.18.0
+frozenlist==1.7.0
+networkx==3.4.2
+importlib_resources==6.5.2
+platformdirs==4.3.8
+nvidia-cudnn-cu12==9.5.1.17
+torchmetrics==1.7.3
+torch==2.7.1
+Pygments==2.19.1
+python-slugify==8.0.4
+jsonargparse==4.40.0
+propcache==0.3.2
+GitPython==3.1.44
+markdown-it-py==3.0.0
+setuptools==78.1.1
+nvidia-curand-cu12==10.3.7.77
+click==8.2.1

checkpoint/no/AttR2Unet/wandb_logs/wandb/run-20250630_035845-4hs9k1ow/files/wandb-metadata.json

@@ -0,0 +1,95 @@
+{
+  "os": "Linux-5.4.0-121-generic-x86_64-with-glibc2.31",
+  "python": "CPython 3.10.18",
+  "startedAt": "2025-06-29T20:58:45.697862Z",
+  "args": [
+    "--config",
+    "configs/AttR2Unet.yaml"
+  ],
+  "program": "/home/vinhbk1/weatherforecast/unet/src/train.py",
+  "codePath": "src/train.py",
+  "email": "weatherforecast1024hcmut@gmail.com",
+  "root": "checkpoint/AttR2Unet/wandb_logs",
+  "host": "ithndgx005",
+  "executable": "/cm/shared/miniconda3/envs/unet/bin/python",
+  "codePathLocal": "src/train.py",
+  "cpu_count": 128,
+  "cpu_count_logical": 256,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "gpu_count": 8,
+  "disk": {
+    "/": {
+      "total": "1888635834368",
+      "used": "54302867456"
+    }
+  },
+  "memory": {
+    "total": "2164326477824"
+  },
+  "cpu": {
+    "count": 128,
+    "countLogical": 256
+  },
+  "gpu_nvidia": [
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-5e7b14fe-921c-ba36-2aeb-1a5825916bdd"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-ae3cfea4-20a5-e014-8450-3d72c9d20c37"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-63b9f644-f11f-db81-3160-40e87114bdfc"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f38d21de-e2a4-9897-5d9a-c6c3fed95c6e"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-8450c54c-f86b-790b-28a4-c33c4d0ecf4b"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-a18dcce2-8b4b-0565-31b4-eb297b1cc4bb"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-46ccbd8d-c84f-32ae-07c3-675928ac6285"
+    },
+    {
+      "name": "NVIDIA A100-SXM4-80GB",
+      "memoryTotal": "85899345920",
+      "cudaCores": 6912,
+      "architecture": "Ampere",
+      "uuid": "GPU-f842f1aa-b36c-40c9-f7ba-095a709c8cb8"
+    }
+  ],
+  "slurm": {
+    "conf": "/cm/shared/apps/slurm/var/etc/slurm/slurm.conf"
+  },
+  "cudaVersion": "12.3"
+}