diff --git a/.gitattributes b/.gitattributes index a6344aac8c09253b3b630fb776ae94478aa0275b..fdfb74761af130b1a296eb18dfb5ef0407031607 100644 --- a/.gitattributes +++ b/.gitattributes @@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text *.zip filter=lfs diff=lfs merge=lfs -text *.zst filter=lfs diff=lfs merge=lfs -text *tfevents* filter=lfs diff=lfs merge=lfs -text +wandb/offline-run-20240509_110904-sc0bbka3/run-sc0bbka3.wandb filter=lfs diff=lfs merge=lfs -text +wandb/offline-run-20240509_113410-b7nw78zg/run-b7nw78zg.wandb filter=lfs diff=lfs merge=lfs -text diff --git a/checkpoints/latest-checkpoint.pt b/checkpoints/latest-checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..3ecb5a6956a1c7b414b21fa49fdf46cc29988a9d --- /dev/null +++ b/checkpoints/latest-checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:78c5f373d3a608e8905521d81d0c89aee91e2b3c527e76031ceaad492a14051b +size 27039790247 diff --git a/config.json b/config.json new file mode 100644 index 0000000000000000000000000000000000000000..3bc8078f1485160b575b062f72bcce1cd0d818d7 --- /dev/null +++ b/config.json @@ -0,0 +1,55 @@ +{ + "dataset": { + "align_stage_components": [ + "download/med-v0.1-instruct/med_v0_1_mix.json", + "download/med-v0.1-instruct" + ], + "dataset_id": "med-instruct", + "dataset_root_dir": "data", + "finetune_stage_components": [ + "download/med-v0.1-instruct/med_v0_1_mix.json", + "download/med-v0.1-instruct" + ], + "type": "med-instruct" + }, + "model": { + "align_epochs": 1, + "align_global_batch_size": 256, + "align_learning_rate": 0.001, + "align_lr_scheduler_type": "linear-warmup+cosine-decay", + "align_max_grad_norm": 1.0, + "align_max_steps": null, + "align_per_device_batch_size": 16, + "align_train_strategy": "fsdp-shard-grad-op", + "align_warmup_ratio": 0.03, + "align_weight_decay": 0.0, + "arch_specifier": "no-align+gelu-mlp", + "enable_gradient_checkpointing": true, + "enable_mixed_precision_training": true, + "finetune_epochs": 5, + "finetune_global_batch_size": 128, + "finetune_learning_rate": 2e-06, + "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", + "finetune_max_grad_norm": 1.0, + "finetune_max_steps": null, + "finetune_per_device_batch_size": 16, + "finetune_train_strategy": "fsdp-full-shard", + "finetune_warmup_ratio": 0.03, + "finetune_weight_decay": 0.1, + "image_resize_strategy": "letterbox", + "llm_backbone_id": "llama2-7b-chat", + "llm_max_length": 2048, + "model_id": "testmodel", + "reduce_in_full_precision": false, + "type": "one-stage+7b", + "vision_backbone_id": "clip-vit-l-336px" + }, + "pretrained_checkpoint": null, + "run_id": "med-instruct+testmodel+stage-finetune+x7", + "run_root_dir": "runs", + "seed": 7, + "stage": "finetune", + "trackers": [], + "wandb_entity": "stanford-voltron", + "wandb_project": "onyx-vlms" +} \ No newline at end of file diff --git a/config.yaml b/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..f935a240b9c7ebc5838f12b386a8a5b4119306cf --- /dev/null +++ b/config.yaml @@ -0,0 +1,49 @@ +dataset: + align_stage_components: + - download/med-v0.1-instruct/med_v0_1_mix.json + - download/med-v0.1-instruct + dataset_id: med-instruct + dataset_root_dir: data + finetune_stage_components: + - download/med-v0.1-instruct/med_v0_1_mix.json + - download/med-v0.1-instruct + type: med-instruct +model: + align_epochs: 1 + align_global_batch_size: 256 + align_learning_rate: 0.001 + align_lr_scheduler_type: linear-warmup+cosine-decay + align_max_grad_norm: 1.0 + align_max_steps: null + align_per_device_batch_size: 16 + align_train_strategy: fsdp-shard-grad-op + align_warmup_ratio: 0.03 + align_weight_decay: 0.0 + arch_specifier: no-align+gelu-mlp + enable_gradient_checkpointing: true + enable_mixed_precision_training: true + finetune_epochs: 5 + finetune_global_batch_size: 128 + finetune_learning_rate: 2.0e-06 + finetune_lr_scheduler_type: linear-warmup+cosine-decay + finetune_max_grad_norm: 1.0 + finetune_max_steps: null + finetune_per_device_batch_size: 16 + finetune_train_strategy: fsdp-full-shard + finetune_warmup_ratio: 0.03 + finetune_weight_decay: 0.1 + image_resize_strategy: letterbox + llm_backbone_id: llama2-7b-chat + llm_max_length: 2048 + model_id: testmodel + reduce_in_full_precision: false + type: one-stage+7b + vision_backbone_id: clip-vit-l-336px +pretrained_checkpoint: null +run_id: med-instruct+testmodel+stage-finetune+x7 +run_root_dir: runs +seed: 7 +stage: finetune +trackers: [] +wandb_entity: stanford-voltron +wandb_project: onyx-vlms diff --git a/latest-checkpoint.pt b/latest-checkpoint.pt new file mode 100644 index 0000000000000000000000000000000000000000..cd28df05f1a17dee4794ac83db9ab14c9a3dfe74 --- /dev/null +++ b/latest-checkpoint.pt @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ff08fedd7524b6ef6bd591dccae8ed88bd040a3acf5edf1e25dd3e54d2399f90 +size 27035595943 diff --git a/med-instruct+testmodel+stage-finetune+x7.jsonl b/med-instruct+testmodel+stage-finetune+x7.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..1ff32aa8a87c4e5a3c7e5588fa57a86c0ed82cad --- /dev/null +++ b/med-instruct+testmodel+stage-finetune+x7.jsonl @@ -0,0 +1,244 @@ +{"Finetune/Learning Rate": 1.9941379571543597e-05, "Finetune/Loss": 7.36778450012207, "Finetune/Loss (Raw)": 7.36778450012207, "Finetune/Step": 1, "Finetune/Step Time": 7.549227952957153} +{"Finetune/Learning Rate": 1.976620555710087e-05, "Finetune/Loss": 7.5681657791137695, "Finetune/Loss (Raw)": 7.768547534942627, "Finetune/Step": 2, "Finetune/Step Time": 4.983920335769653} +{"Finetune/Learning Rate": 1.9476531711828027e-05, "Finetune/Loss": 6.51008939743042, "Finetune/Loss (Raw)": 4.3939361572265625, "Finetune/Step": 3, "Finetune/Step Time": 4.107757012049357} +{"Finetune/Learning Rate": 1.9075754196709574e-05, "Finetune/Loss": 5.771026611328125, "Finetune/Loss (Raw)": 3.553839921951294, "Finetune/Step": 4, "Finetune/Step Time": 3.7063944339752197} +{"Finetune/Learning Rate": 1.8568571761675893e-05, "Finetune/Loss": 5.1307196617126465, "Finetune/Loss (Raw)": 2.5694918632507324, "Finetune/Step": 5, "Finetune/Step Time": 3.413467216491699} +{"Finetune/Learning Rate": 1.796093065705644e-05, "Finetune/Loss": 4.744967937469482, "Finetune/Loss (Raw)": 2.8162074089050293, "Finetune/Step": 6, "Finetune/Step Time": 3.2199769020080566} +{"Finetune/Learning Rate": 1.725995491923131e-05, "Finetune/Loss": 4.239523410797119, "Finetune/Loss (Raw)": 1.2068538665771484, "Finetune/Step": 7, "Finetune/Step Time": 3.080077750342233} +{"Finetune/Learning Rate": 1.647386284781828e-05, "Finetune/Loss": 3.892263650894165, "Finetune/Loss (Raw)": 1.4614497423171997, "Finetune/Step": 8, "Finetune/Step Time": 3.0190326869487762} +{"Finetune/Learning Rate": 1.5611870653623826e-05, "Finetune/Loss": 3.5654280185699463, "Finetune/Loss (Raw)": 0.9507445096969604, "Finetune/Step": 9, "Finetune/Step Time": 2.93222983678182} +{"Finetune/Learning Rate": 1.4684084406997903e-05, "Finetune/Loss": 3.348705291748047, "Finetune/Loss (Raw)": 1.3981952667236328, "Finetune/Step": 10, "Finetune/Step Time": 2.863704705238342} +{"Finetune/Learning Rate": 1.3701381553399147e-05, "Finetune/Loss": 3.1944899559020996, "Finetune/Loss (Raw)": 1.6523410081863403, "Finetune/Step": 11, "Finetune/Step Time": 2.825130484320901} +{"Finetune/Learning Rate": 1.2675283385292212e-05, "Finetune/Loss": 3.010889768600464, "Finetune/Loss (Raw)": 0.9912872910499573, "Finetune/Step": 12, "Finetune/Step Time": 2.785505453745524} +{"Finetune/Learning Rate": 1.161781996552765e-05, "Finetune/Loss": 2.882206916809082, "Finetune/Loss (Raw)": 1.3380084037780762, "Finetune/Step": 13, "Finetune/Step Time": 2.7438870760110707} +{"Finetune/Learning Rate": 1.0541389085854177e-05, "Finetune/Loss": 2.7508015632629395, "Finetune/Loss (Raw)": 1.0425317287445068, "Finetune/Step": 14, "Finetune/Step Time": 2.7094162702560425} +{"Finetune/Learning Rate": 9.458610914145826e-06, "Finetune/Loss": 2.6125948429107666, "Finetune/Loss (Raw)": 0.6776995062828064, "Finetune/Step": 15, "Finetune/Step Time": 2.6983984470367433} +{"Finetune/Learning Rate": 8.382180034472353e-06, "Finetune/Loss": 2.53676438331604, "Finetune/Loss (Raw)": 1.3993110656738281, "Finetune/Step": 16, "Finetune/Step Time": 2.670434609055519} +{"Finetune/Learning Rate": 7.324716614707794e-06, "Finetune/Loss": 2.4528164863586426, "Finetune/Loss (Raw)": 1.1096470355987549, "Finetune/Step": 17, "Finetune/Step Time": 2.645362601560705} +{"Finetune/Learning Rate": 6.298618446600856e-06, "Finetune/Loss": 2.3574817180633545, "Finetune/Loss (Raw)": 0.7367918491363525, "Finetune/Step": 18, "Finetune/Step Time": 2.6233630180358887} +{"Finetune/Learning Rate": 5.3159155930021e-06, "Finetune/Loss": 2.281595468521118, "Finetune/Loss (Raw)": 0.9156460762023926, "Finetune/Step": 19, "Finetune/Step Time": 2.612993039582905} +{"Finetune/Learning Rate": 4.388129346376177e-06, "Finetune/Loss": 2.2257227897644043, "Finetune/Loss (Raw)": 1.1641401052474976, "Finetune/Step": 20, "Finetune/Step Time": 2.6002017498016357} +{"Finetune/Learning Rate": 3.5261371521817247e-06, "Finetune/Loss": 2.172165870666504, "Finetune/Loss (Raw)": 1.1010304689407349, "Finetune/Step": 21, "Finetune/Step Time": 2.5839241913386752} +{"Finetune/Learning Rate": 2.740045080768694e-06, "Finetune/Loss": 2.137272357940674, "Finetune/Loss (Raw)": 1.4045038223266602, "Finetune/Step": 22, "Finetune/Step Time": 2.568736423145641} +{"Finetune/Learning Rate": 2.0390693429435626e-06, "Finetune/Loss": 2.085879325866699, "Finetune/Loss (Raw)": 0.9552371501922607, "Finetune/Step": 23, "Finetune/Step Time": 2.5667630796847134} +{"Finetune/Learning Rate": 1.4314282383241097e-06, "Finetune/Loss": 2.017505168914795, "Finetune/Loss (Raw)": 0.44490066170692444, "Finetune/Step": 24, "Finetune/Step Time": 2.553176204363505} +{"Finetune/Learning Rate": 9.242458032904311e-07, "Finetune/Loss": 1.9853742122650146, "Finetune/Loss (Raw)": 1.2142295837402344, "Finetune/Step": 25, "Finetune/Step Time": 2.5411977481842043} +{"Finetune/Learning Rate": 5.234682881719766e-07, "Finetune/Loss": 1.9280644655227661, "Finetune/Loss (Raw)": 0.49531811475753784, "Finetune/Step": 26, "Finetune/Step Time": 2.5300285082597} +{"Finetune/Learning Rate": 2.3379444289913344e-07, "Finetune/Loss": 1.8876720666885376, "Finetune/Loss (Raw)": 0.8374679088592529, "Finetune/Step": 27, "Finetune/Step Time": 2.5307990974850125} +{"Finetune/Learning Rate": 5.862042845640403e-08, "Finetune/Loss": 1.8572579622268677, "Finetune/Loss (Raw)": 1.036075234413147, "Finetune/Step": 28, "Finetune/Step Time": 2.520767722811018} +{"Finetune/Learning Rate": 0.0, "Finetune/Loss": 1.8288624286651611, "Finetune/Loss (Raw)": 1.0337905883789062, "Finetune/Step": 29, "Finetune/Step Time": 2.5119694183612693} +{"Finetune/Learning Rate": 1.9941379571543597e-05, "Finetune/Loss": 7.36778450012207, "Finetune/Loss (Raw)": 7.36778450012207, "Finetune/Step": 1, "Finetune/Step Time": 9.052452564239502} +{"Finetune/Learning Rate": 1.976620555710087e-05, "Finetune/Loss": 7.5681657791137695, "Finetune/Loss (Raw)": 7.768547534942627, "Finetune/Step": 2, "Finetune/Step Time": 5.9602580070495605} +{"Finetune/Learning Rate": 1.9476531711828027e-05, "Finetune/Loss": 6.511626243591309, "Finetune/Loss (Raw)": 4.398545265197754, "Finetune/Step": 3, "Finetune/Step Time": 4.841636419296265} +{"Finetune/Learning Rate": 1.9075754196709574e-05, "Finetune/Loss": 5.770370960235596, "Finetune/Loss (Raw)": 3.54660701751709, "Finetune/Step": 4, "Finetune/Step Time": 4.321389198303223} +{"Finetune/Learning Rate": 1.8568571761675893e-05, "Finetune/Loss": 5.130580902099609, "Finetune/Loss (Raw)": 2.571420907974243, "Finetune/Step": 5, "Finetune/Step Time": 3.945681428909302} +{"Finetune/Learning Rate": 1.796093065705644e-05, "Finetune/Loss": 4.744907379150391, "Finetune/Loss (Raw)": 2.8165385723114014, "Finetune/Step": 6, "Finetune/Step Time": 3.6996482610702515} +{"Finetune/Learning Rate": 1.725995491923131e-05, "Finetune/Loss": 4.240437984466553, "Finetune/Loss (Raw)": 1.2136189937591553, "Finetune/Step": 7, "Finetune/Step Time": 3.5218572275979176} +{"Finetune/Learning Rate": 1.647386284781828e-05, "Finetune/Loss": 3.8936822414398193, "Finetune/Loss (Raw)": 1.4663951396942139, "Finetune/Step": 8, "Finetune/Step Time": 3.4337348341941833} +{"Finetune/Learning Rate": 1.5611870653623826e-05, "Finetune/Loss": 3.5675957202911377, "Finetune/Loss (Raw)": 0.9588993787765503, "Finetune/Step": 9, "Finetune/Step Time": 3.324105289247301} +{"Finetune/Learning Rate": 1.4684084406997903e-05, "Finetune/Loss": 3.346062421798706, "Finetune/Loss (Raw)": 1.352268099784851, "Finetune/Step": 10, "Finetune/Step Time": 3.238165497779846} +{"Finetune/Learning Rate": 1.3701381553399147e-05, "Finetune/Loss": 3.1957342624664307, "Finetune/Loss (Raw)": 1.692449688911438, "Finetune/Step": 11, "Finetune/Step Time": 3.178570964119651} +{"Finetune/Learning Rate": 1.2675283385292212e-05, "Finetune/Loss": 3.014349937438965, "Finetune/Loss (Raw)": 1.0191234350204468, "Finetune/Step": 12, "Finetune/Step Time": 3.138482451438904} +{"Finetune/Learning Rate": 1.161781996552765e-05, "Finetune/Loss": 2.8854904174804688, "Finetune/Loss (Raw)": 1.339175820350647, "Finetune/Step": 13, "Finetune/Step Time": 3.084709882736206} +{"Finetune/Learning Rate": 1.0541389085854177e-05, "Finetune/Loss": 2.754392147064209, "Finetune/Loss (Raw)": 1.0501140356063843, "Finetune/Step": 14, "Finetune/Step Time": 3.0392286607197354} +{"Finetune/Learning Rate": 9.458610914145826e-06, "Finetune/Loss": 2.6173884868621826, "Finetune/Loss (Raw)": 0.6993362307548523, "Finetune/Step": 15, "Finetune/Step Time": 3.007705497741699} +{"Finetune/Learning Rate": 8.382180034472353e-06, "Finetune/Loss": 2.5399088859558105, "Finetune/Loss (Raw)": 1.3777189254760742, "Finetune/Step": 16, "Finetune/Step Time": 2.987220421433449} +{"Finetune/Learning Rate": 7.324716614707794e-06, "Finetune/Loss": 2.45613169670105, "Finetune/Loss (Raw)": 1.1156938076019287, "Finetune/Step": 17, "Finetune/Step Time": 2.9548792839050293} +{"Finetune/Learning Rate": 6.298618446600856e-06, "Finetune/Loss": 2.359598398208618, "Finetune/Loss (Raw)": 0.7185351848602295, "Finetune/Step": 18, "Finetune/Step Time": 2.925637443860372} +{"Finetune/Learning Rate": 5.3159155930021e-06, "Finetune/Loss": 2.2822110652923584, "Finetune/Loss (Raw)": 0.8892373442649841, "Finetune/Step": 19, "Finetune/Step Time": 2.9009793055684945} +{"Finetune/Learning Rate": 4.388129346376177e-06, "Finetune/Loss": 2.2258594036102295, "Finetune/Loss (Raw)": 1.155174970626831, "Finetune/Step": 20, "Finetune/Step Time": 2.8964689135551454} +{"Finetune/Learning Rate": 3.5261371521817247e-06, "Finetune/Loss": 2.1718244552612305, "Finetune/Loss (Raw)": 1.0911266803741455, "Finetune/Step": 21, "Finetune/Step Time": 2.8752532686506} +{"Finetune/Learning Rate": 2.740045080768694e-06, "Finetune/Loss": 2.137608289718628, "Finetune/Loss (Raw)": 1.4190702438354492, "Finetune/Step": 22, "Finetune/Step Time": 2.855891141024503} +{"Finetune/Learning Rate": 2.0390693429435626e-06, "Finetune/Loss": 2.084510087966919, "Finetune/Loss (Raw)": 0.9163495898246765, "Finetune/Step": 23, "Finetune/Step Time": 2.8430409327797266} +{"Finetune/Learning Rate": 1.4314282383241097e-06, "Finetune/Loss": 2.0148420333862305, "Finetune/Loss (Raw)": 0.4124765992164612, "Finetune/Step": 24, "Finetune/Step Time": 2.835821568965912} +{"Finetune/Learning Rate": 9.242458032904311e-07, "Finetune/Loss": 1.9838645458221436, "Finetune/Loss (Raw)": 1.2404075860977173, "Finetune/Step": 25, "Finetune/Step Time": 2.8206040763854983} +{"Finetune/Learning Rate": 5.234682881719766e-07, "Finetune/Loss": 1.9279828071594238, "Finetune/Loss (Raw)": 0.5309382081031799, "Finetune/Step": 26, "Finetune/Step Time": 2.8069500648058376} +{"Finetune/Learning Rate": 2.3379444289913344e-07, "Finetune/Loss": 1.8888658285140991, "Finetune/Loss (Raw)": 0.8718276023864746, "Finetune/Step": 27, "Finetune/Step Time": 2.7932933701409235} +{"Finetune/Learning Rate": 5.862042845640403e-08, "Finetune/Loss": 1.8589668273925781, "Finetune/Loss (Raw)": 1.0516877174377441, "Finetune/Step": 28, "Finetune/Step Time": 2.79367356640952} +{"Finetune/Learning Rate": 0.0, "Finetune/Loss": 1.8314627408981323, "Finetune/Loss (Raw)": 1.0613527297973633, "Finetune/Step": 29, "Finetune/Step Time": 2.7814989912098853} +{"Finetune/Learning Rate": 4.878048780487805e-07, "Finetune/Loss": 4.9622297286987305, "Finetune/Loss (Raw)": 4.9622297286987305, "Finetune/Step": 1, "Finetune/Step Time": 18.78721833229065} +{"Finetune/Learning Rate": 9.75609756097561e-07, "Finetune/Loss": 4.813885688781738, "Finetune/Loss (Raw)": 4.665542125701904, "Finetune/Step": 2, "Finetune/Step Time": 14.126603603363037} +{"Finetune/Learning Rate": 1.4634146341463414e-06, "Finetune/Loss": 4.7901201248168945, "Finetune/Loss (Raw)": 4.742588043212891, "Finetune/Step": 3, "Finetune/Step Time": 12.41076405843099} +{"Finetune/Learning Rate": 1.951219512195122e-06, "Finetune/Loss": 4.706673622131348, "Finetune/Loss (Raw)": 4.456334590911865, "Finetune/Step": 4, "Finetune/Step Time": 11.567386388778687} +{"Finetune/Learning Rate": 2.4390243902439027e-06, "Finetune/Loss": 4.610179424285889, "Finetune/Loss (Raw)": 4.224202632904053, "Finetune/Step": 5, "Finetune/Step Time": 11.030108737945557} +{"Finetune/Learning Rate": 2.926829268292683e-06, "Finetune/Loss": 4.413971424102783, "Finetune/Loss (Raw)": 3.432929277420044, "Finetune/Step": 6, "Finetune/Step Time": 10.681593298912048} +{"Finetune/Learning Rate": 3.414634146341464e-06, "Finetune/Loss": 4.168725967407227, "Finetune/Loss (Raw)": 2.6972522735595703, "Finetune/Step": 7, "Finetune/Step Time": 10.490775755473546} +{"Finetune/Learning Rate": 3.902439024390244e-06, "Finetune/Loss": 3.8979344367980957, "Finetune/Loss (Raw)": 2.00239634513855, "Finetune/Step": 8, "Finetune/Step Time": 10.374537259340286} +{"Finetune/Learning Rate": 4.390243902439025e-06, "Finetune/Loss": 3.659745931625366, "Finetune/Loss (Raw)": 1.7542362213134766, "Finetune/Step": 9, "Finetune/Step Time": 10.269062386618721} +{"Finetune/Learning Rate": 4.8780487804878055e-06, "Finetune/Loss": 3.3994293212890625, "Finetune/Loss (Raw)": 1.0565803050994873, "Finetune/Step": 10, "Finetune/Step Time": 10.112186241149903} +{"Finetune/Learning Rate": 5.365853658536586e-06, "Finetune/Loss": 3.162954568862915, "Finetune/Loss (Raw)": 0.7982063293457031, "Finetune/Step": 11, "Finetune/Step Time": 10.018468488346446} +{"Finetune/Learning Rate": 5.853658536585366e-06, "Finetune/Loss": 2.9502322673797607, "Finetune/Loss (Raw)": 0.6102874279022217, "Finetune/Step": 12, "Finetune/Step Time": 9.944100121657053} +{"Finetune/Learning Rate": 6.341463414634147e-06, "Finetune/Loss": 2.757354259490967, "Finetune/Loss (Raw)": 0.4428159296512604, "Finetune/Step": 13, "Finetune/Step Time": 9.953564588840191} +{"Finetune/Learning Rate": 6.829268292682928e-06, "Finetune/Loss": 2.587533950805664, "Finetune/Loss (Raw)": 0.379874587059021, "Finetune/Step": 14, "Finetune/Step Time": 9.932186211858477} +{"Finetune/Learning Rate": 7.317073170731707e-06, "Finetune/Loss": 2.4365665912628174, "Finetune/Loss (Raw)": 0.3230189383029938, "Finetune/Step": 15, "Finetune/Step Time": 9.89031270345052} +{"Finetune/Learning Rate": 7.804878048780489e-06, "Finetune/Loss": 2.3064534664154053, "Finetune/Loss (Raw)": 0.3547614812850952, "Finetune/Step": 16, "Finetune/Step Time": 9.873847678303719} +{"Finetune/Learning Rate": 8.292682926829268e-06, "Finetune/Loss": 2.1886589527130127, "Finetune/Loss (Raw)": 0.3039456605911255, "Finetune/Step": 17, "Finetune/Step Time": 9.825686538920683} +{"Finetune/Learning Rate": 8.78048780487805e-06, "Finetune/Loss": 2.0806972980499268, "Finetune/Loss (Raw)": 0.24534659087657928, "Finetune/Step": 18, "Finetune/Step Time": 9.80287222067515} +{"Finetune/Learning Rate": 9.268292682926831e-06, "Finetune/Loss": 1.9838893413543701, "Finetune/Loss (Raw)": 0.241348534822464, "Finetune/Step": 19, "Finetune/Step Time": 9.789240573581896} +{"Finetune/Learning Rate": 9.756097560975611e-06, "Finetune/Loss": 1.8947786092758179, "Finetune/Loss (Raw)": 0.20167773962020874, "Finetune/Step": 20, "Finetune/Step Time": 9.78241444826126} +{"Finetune/Learning Rate": 1.024390243902439e-05, "Finetune/Loss": 1.8145203590393066, "Finetune/Loss (Raw)": 0.20935341715812683, "Finetune/Step": 21, "Finetune/Step Time": 9.745673281805855} +{"Finetune/Learning Rate": 1.0731707317073172e-05, "Finetune/Loss": 1.7402572631835938, "Finetune/Loss (Raw)": 0.1807323694229126, "Finetune/Step": 22, "Finetune/Step Time": 9.73571987585588} +{"Finetune/Learning Rate": 1.1219512195121953e-05, "Finetune/Loss": 1.6731915473937988, "Finetune/Loss (Raw)": 0.19774509966373444, "Finetune/Step": 23, "Finetune/Step Time": 9.697343370188838} +{"Finetune/Learning Rate": 1.1707317073170731e-05, "Finetune/Loss": 1.611534833908081, "Finetune/Loss (Raw)": 0.1934271901845932, "Finetune/Step": 24, "Finetune/Step Time": 9.664562821388245} +{"Finetune/Learning Rate": 1.2195121951219513e-05, "Finetune/Loss": 1.5547436475753784, "Finetune/Loss (Raw)": 0.19175826013088226, "Finetune/Step": 25, "Finetune/Step Time": 9.65188998222351} +{"Finetune/Learning Rate": 1.2682926829268294e-05, "Finetune/Loss": 1.5038423538208008, "Finetune/Loss (Raw)": 0.23131072521209717, "Finetune/Step": 26, "Finetune/Step Time": 9.647450667161207} +{"Finetune/Learning Rate": 1.3170731707317076e-05, "Finetune/Loss": 1.4548784494400024, "Finetune/Loss (Raw)": 0.18181724846363068, "Finetune/Step": 27, "Finetune/Step Time": 9.608060518900553} +{"Finetune/Learning Rate": 1.3658536585365855e-05, "Finetune/Loss": 1.408819556236267, "Finetune/Loss (Raw)": 0.1652291715145111, "Finetune/Step": 28, "Finetune/Step Time": 9.587811725480217} +{"Finetune/Learning Rate": 1.4146341463414635e-05, "Finetune/Loss": 1.3664113283157349, "Finetune/Loss (Raw)": 0.1789785623550415, "Finetune/Step": 29, "Finetune/Step Time": 9.570461651374554} +{"Finetune/Learning Rate": 1.4634146341463415e-05, "Finetune/Loss": 1.326568603515625, "Finetune/Loss (Raw)": 0.17112748324871063, "Finetune/Step": 30, "Finetune/Step Time": 9.54690367380778} +{"Finetune/Learning Rate": 1.5121951219512196e-05, "Finetune/Loss": 1.289267659187317, "Finetune/Loss (Raw)": 0.17024636268615723, "Finetune/Step": 31, "Finetune/Step Time": 9.534612155729725} +{"Finetune/Learning Rate": 1.5609756097560978e-05, "Finetune/Loss": 1.2538834810256958, "Finetune/Loss (Raw)": 0.15697208046913147, "Finetune/Step": 32, "Finetune/Step Time": 9.542723596096039} +{"Finetune/Learning Rate": 1.6097560975609757e-05, "Finetune/Loss": 1.1042906045913696, "Finetune/Loss (Raw)": 0.17525750398635864, "Finetune/Step": 33, "Finetune/Step Time": 9.538711995789498} +{"Finetune/Learning Rate": 1.6585365853658537e-05, "Finetune/Loss": 0.9638158082962036, "Finetune/Loss (Raw)": 0.1703479290008545, "Finetune/Step": 34, "Finetune/Step Time": 9.529267879093394} +{"Finetune/Learning Rate": 1.7073170731707317e-05, "Finetune/Loss": 0.8200684785842896, "Finetune/Loss (Raw)": 0.1426728367805481, "Finetune/Step": 35, "Finetune/Step Time": 9.521432318006243} +{"Finetune/Learning Rate": 1.75609756097561e-05, "Finetune/Loss": 0.6854739785194397, "Finetune/Loss (Raw)": 0.14931100606918335, "Finetune/Step": 36, "Finetune/Step Time": 9.504447784688738} +{"Finetune/Learning Rate": 1.804878048780488e-05, "Finetune/Loss": 0.5583795309066772, "Finetune/Loss (Raw)": 0.1571805775165558, "Finetune/Step": 37, "Finetune/Step Time": 9.496721725206118} +{"Finetune/Learning Rate": 1.0000000000000002e-06, "Finetune/Loss": 4.711216926574707, "Finetune/Loss (Raw)": 4.711216926574707, "Finetune/Step": 1, "Finetune/Step Time": 22.233684301376343} +{"Finetune/Learning Rate": 2.0000000000000003e-06, "Finetune/Loss": 4.707284927368164, "Finetune/Loss (Raw)": 4.703352928161621, "Finetune/Step": 2, "Finetune/Step Time": 20.681366443634033} +{"Finetune/Learning Rate": 3e-06, "Finetune/Loss": 4.67268180847168, "Finetune/Loss (Raw)": 4.6034746170043945, "Finetune/Step": 3, "Finetune/Step Time": 19.463648557662964} +{"Finetune/Learning Rate": 4.000000000000001e-06, "Finetune/Loss": 4.508688449859619, "Finetune/Loss (Raw)": 4.016709327697754, "Finetune/Step": 4, "Finetune/Step Time": 18.97678828239441} +{"Finetune/Learning Rate": 5e-06, "Finetune/Loss": 4.204005718231201, "Finetune/Loss (Raw)": 2.9852728843688965, "Finetune/Step": 5, "Finetune/Step Time": 18.612454891204834} +{"Finetune/Learning Rate": 6e-06, "Finetune/Loss": 3.825981616973877, "Finetune/Loss (Raw)": 1.9358625411987305, "Finetune/Step": 6, "Finetune/Step Time": 18.293881972630818} +{"Finetune/Learning Rate": 7e-06, "Finetune/Loss": 3.5160574913024902, "Finetune/Loss (Raw)": 1.6565135717391968, "Finetune/Step": 7, "Finetune/Step Time": 18.225826127188547} +{"Finetune/Learning Rate": 8.000000000000001e-06, "Finetune/Loss": 3.197500467300415, "Finetune/Loss (Raw)": 0.9676005244255066, "Finetune/Step": 8, "Finetune/Step Time": 18.142492443323135} +{"Finetune/Learning Rate": 9e-06, "Finetune/Loss": 2.9274792671203613, "Finetune/Loss (Raw)": 0.7673105597496033, "Finetune/Step": 9, "Finetune/Step Time": 18.01582709948222} +{"Finetune/Learning Rate": 1e-05, "Finetune/Loss": 2.6797354221343994, "Finetune/Loss (Raw)": 0.45003974437713623, "Finetune/Step": 10, "Finetune/Step Time": 17.905010485649107} +{"Finetune/Learning Rate": 1.1000000000000001e-05, "Finetune/Loss": 2.471994400024414, "Finetune/Loss (Raw)": 0.39458468556404114, "Finetune/Step": 11, "Finetune/Step Time": 17.80310286175121} +{"Finetune/Learning Rate": 1.2e-05, "Finetune/Loss": 2.2968461513519287, "Finetune/Loss (Raw)": 0.37021592259407043, "Finetune/Step": 12, "Finetune/Step Time": 17.73170703649521} +{"Finetune/Learning Rate": 1.3000000000000001e-05, "Finetune/Loss": 2.142801523208618, "Finetune/Loss (Raw)": 0.2942649722099304, "Finetune/Step": 13, "Finetune/Step Time": 17.71355731670673} +{"Finetune/Learning Rate": 1.4e-05, "Finetune/Loss": 2.0097575187683105, "Finetune/Loss (Raw)": 0.2801855206489563, "Finetune/Step": 14, "Finetune/Step Time": 17.68396689210619} +{"Finetune/Learning Rate": 1.5000000000000002e-05, "Finetune/Loss": 1.895301103591919, "Finetune/Loss (Raw)": 0.2929112911224365, "Finetune/Step": 15, "Finetune/Step Time": 17.63153033256531} +{"Finetune/Learning Rate": 1.6000000000000003e-05, "Finetune/Loss": 1.791799545288086, "Finetune/Loss (Raw)": 0.23927782475948334, "Finetune/Step": 16, "Finetune/Step Time": 17.625697061419487} +{"Finetune/Learning Rate": 1.7e-05, "Finetune/Loss": 1.698999047279358, "Finetune/Loss (Raw)": 0.21418961882591248, "Finetune/Step": 17, "Finetune/Step Time": 17.635714713264914} +{"Finetune/Learning Rate": 1.8e-05, "Finetune/Loss": 1.6166439056396484, "Finetune/Loss (Raw)": 0.21660563349723816, "Finetune/Step": 18, "Finetune/Step Time": 17.599896483951145} +{"Finetune/Learning Rate": 1.9e-05, "Finetune/Loss": 1.5405137538909912, "Finetune/Loss (Raw)": 0.17017090320587158, "Finetune/Step": 19, "Finetune/Step Time": 17.63633697911313} +{"Finetune/Learning Rate": 2e-05, "Finetune/Loss": 1.4720044136047363, "Finetune/Loss (Raw)": 0.17032840847969055, "Finetune/Step": 20, "Finetune/Step Time": 17.63486514091492} +{"Finetune/Learning Rate": 1.999988940995767e-05, "Finetune/Loss": 1.4106765985488892, "Finetune/Loss (Raw)": 0.18412093818187714, "Finetune/Step": 21, "Finetune/Step Time": 17.60664134933835} +{"Finetune/Learning Rate": 1.9999557642276715e-05, "Finetune/Loss": 1.3553168773651123, "Finetune/Loss (Raw)": 0.19276167452335358, "Finetune/Step": 22, "Finetune/Step Time": 17.566573988307606} +{"Finetune/Learning Rate": 1.9999004704295167e-05, "Finetune/Loss": 1.303739309310913, "Finetune/Loss (Raw)": 0.16903311014175415, "Finetune/Step": 23, "Finetune/Step Time": 17.541278465934422} +{"Finetune/Learning Rate": 1.9998230608242917e-05, "Finetune/Loss": 1.2562918663024902, "Finetune/Loss (Raw)": 0.165000781416893, "Finetune/Step": 24, "Finetune/Step Time": 17.55869574348132} +{"Finetune/Learning Rate": 1.999723537124143e-05, "Finetune/Loss": 1.2122657299041748, "Finetune/Loss (Raw)": 0.15564051270484924, "Finetune/Step": 25, "Finetune/Step Time": 17.550940198898317} +{"Finetune/Learning Rate": 1.9996019015303366e-05, "Finetune/Loss": 1.1723536252975464, "Finetune/Loss (Raw)": 0.17454947531223297, "Finetune/Step": 26, "Finetune/Step Time": 17.5770446062088} +{"Finetune/Learning Rate": 1.9994581567332095e-05, "Finetune/Loss": 1.1347756385803223, "Finetune/Loss (Raw)": 0.15774784982204437, "Finetune/Step": 27, "Finetune/Step Time": 17.550292774483008} +{"Finetune/Learning Rate": 1.9992923059121107e-05, "Finetune/Loss": 1.0996936559677124, "Finetune/Loss (Raw)": 0.15247702598571777, "Finetune/Step": 28, "Finetune/Step Time": 17.533284178801946} +{"Finetune/Learning Rate": 1.999104352735329e-05, "Finetune/Loss": 1.0668989419937134, "Finetune/Loss (Raw)": 0.14865100383758545, "Finetune/Step": 29, "Finetune/Step Time": 17.511749711529962} +{"Finetune/Learning Rate": 1.9988943013600156e-05, "Finetune/Loss": 1.0359028577804565, "Finetune/Loss (Raw)": 0.13701483607292175, "Finetune/Step": 30, "Finetune/Step Time": 17.50567758878072} +{"Finetune/Learning Rate": 1.9986621564320878e-05, "Finetune/Loss": 1.0077641010284424, "Finetune/Loss (Raw)": 0.16360332071781158, "Finetune/Step": 31, "Finetune/Step Time": 17.521542518369614} +{"Finetune/Learning Rate": 1.9984079230861296e-05, "Finetune/Loss": 0.9808215498924255, "Finetune/Loss (Raw)": 0.14560051262378693, "Finetune/Step": 32, "Finetune/Step Time": 17.501714952290058} +{"Finetune/Learning Rate": 1.998131606945276e-05, "Finetune/Loss": 0.8383952379226685, "Finetune/Loss (Raw)": 0.1535751223564148, "Finetune/Step": 33, "Finetune/Step Time": 17.485050324237708} +{"Finetune/Learning Rate": 1.9978332141210894e-05, "Finetune/Loss": 0.6954304575920105, "Finetune/Loss (Raw)": 0.12847968935966492, "Finetune/Step": 34, "Finetune/Step Time": 17.465691307011774} +{"Finetune/Learning Rate": 1.997512751213426e-05, "Finetune/Loss": 0.5557042360305786, "Finetune/Loss (Raw)": 0.13223737478256226, "Finetune/Step": 35, "Finetune/Step Time": 17.47207282611302} +{"Finetune/Learning Rate": 1.9971702253102856e-05, "Finetune/Loss": 0.4337284564971924, "Finetune/Loss (Raw)": 0.11348187178373337, "Finetune/Step": 36, "Finetune/Step Time": 17.46778596109814} +{"Finetune/Learning Rate": 1.99680564398766e-05, "Finetune/Loss": 0.3447180390357971, "Finetune/Loss (Raw)": 0.13694092631340027, "Finetune/Step": 37, "Finetune/Step Time": 17.473582609279735} +{"Finetune/Learning Rate": 1.9964190153093613e-05, "Finetune/Loss": 0.28849250078201294, "Finetune/Loss (Raw)": 0.13664454221725464, "Finetune/Step": 38, "Finetune/Step Time": 17.498738263782702} +{"Finetune/Learning Rate": 1.996010347826847e-05, "Finetune/Loss": 0.24090710282325745, "Finetune/Loss (Raw)": 0.13378062844276428, "Finetune/Step": 39, "Finetune/Step Time": 17.48963096202948} +{"Finetune/Learning Rate": 1.9955796505790266e-05, "Finetune/Loss": 0.2148941159248352, "Finetune/Loss (Raw)": 0.1351858377456665, "Finetune/Step": 40, "Finetune/Step Time": 17.48420195579529} +{"Finetune/Learning Rate": 1.9951269330920667e-05, "Finetune/Loss": 0.1943114697933197, "Finetune/Loss (Raw)": 0.10866530984640121, "Finetune/Step": 41, "Finetune/Step Time": 17.468334116586824} +{"Finetune/Learning Rate": 1.9946522053791755e-05, "Finetune/Loss": 0.1838119626045227, "Finetune/Loss (Raw)": 0.11405590176582336, "Finetune/Step": 42, "Finetune/Step Time": 17.465145729836962} +{"Finetune/Learning Rate": 1.9941554779403854e-05, "Finetune/Loss": 0.1752166897058487, "Finetune/Loss (Raw)": 0.11953572183847427, "Finetune/Step": 43, "Finetune/Step Time": 17.472803892091264} +{"Finetune/Learning Rate": 1.993636761762317e-05, "Finetune/Loss": 0.16728492081165314, "Finetune/Loss (Raw)": 0.11639939248561859, "Finetune/Step": 44, "Finetune/Step Time": 17.48548124595122} +{"Finetune/Learning Rate": 1.9930960683179404e-05, "Finetune/Loss": 0.1616811454296112, "Finetune/Loss (Raw)": 0.11494384706020355, "Finetune/Step": 45, "Finetune/Step Time": 17.486545356114707} +{"Finetune/Learning Rate": 1.992533409566317e-05, "Finetune/Loss": 0.15730100870132446, "Finetune/Loss (Raw)": 0.14002111554145813, "Finetune/Step": 46, "Finetune/Step Time": 17.48470501796059} +{"Finetune/Learning Rate": 1.991948797952338e-05, "Finetune/Loss": 0.1521824449300766, "Finetune/Loss (Raw)": 0.12911736965179443, "Finetune/Step": 47, "Finetune/Step Time": 17.46926967133867} +{"Finetune/Learning Rate": 1.991342246406448e-05, "Finetune/Loss": 0.1484510600566864, "Finetune/Loss (Raw)": 0.11987350881099701, "Finetune/Step": 48, "Finetune/Step Time": 17.466019362211227} +{"Finetune/Learning Rate": 1.9907137683443593e-05, "Finetune/Loss": 0.1450246274471283, "Finetune/Loss (Raw)": 0.10454348474740982, "Finetune/Step": 49, "Finetune/Step Time": 17.472221335586237} +{"Finetune/Learning Rate": 1.990063377666755e-05, "Finetune/Loss": 0.1419849395751953, "Finetune/Loss (Raw)": 0.11933600902557373, "Finetune/Step": 50, "Finetune/Step Time": 17.48387352466583} +{"Finetune/Learning Rate": 1.9893910887589817e-05, "Finetune/Loss": 0.13981971144676208, "Finetune/Loss (Raw)": 0.10088327527046204, "Finetune/Step": 51, "Finetune/Step Time": 17.46796452297884} +{"Finetune/Learning Rate": 1.9886969164907306e-05, "Finetune/Loss": 0.13806769251823425, "Finetune/Loss (Raw)": 0.11426441371440887, "Finetune/Step": 52, "Finetune/Step Time": 17.463753049190228} +{"Finetune/Learning Rate": 1.9879808762157106e-05, "Finetune/Loss": 0.13594219088554382, "Finetune/Loss (Raw)": 0.1161043792963028, "Finetune/Step": 53, "Finetune/Step Time": 17.459364103821088} +{"Finetune/Learning Rate": 1.987242983771306e-05, "Finetune/Loss": 0.13352537155151367, "Finetune/Loss (Raw)": 0.11542361974716187, "Finetune/Step": 54, "Finetune/Step Time": 17.452685899204678} +{"Finetune/Learning Rate": 1.986483255478229e-05, "Finetune/Loss": 0.13118571043014526, "Finetune/Loss (Raw)": 0.09416383504867554, "Finetune/Step": 55, "Finetune/Step Time": 17.46624122099443} +{"Finetune/Learning Rate": 1.985701708140155e-05, "Finetune/Loss": 0.1290861964225769, "Finetune/Loss (Raw)": 0.09781654924154282, "Finetune/Step": 56, "Finetune/Step Time": 17.475188719374792} +{"Finetune/Learning Rate": 1.0000000000000002e-06, "Finetune/Loss": 4.718144416809082, "Finetune/Loss (Raw)": 4.718144416809082, "Finetune/Step": 1, "Finetune/Step Time": 21.09275221824646} +{"Finetune/Learning Rate": 2.0000000000000003e-06, "Finetune/Loss": 4.705273628234863, "Finetune/Loss (Raw)": 4.692403316497803, "Finetune/Step": 2, "Finetune/Step Time": 19.911753296852112} +{"Finetune/Learning Rate": 3e-06, "Finetune/Loss": 4.668498992919922, "Finetune/Loss (Raw)": 4.594948768615723, "Finetune/Step": 3, "Finetune/Step Time": 19.282703320185345} +{"Finetune/Learning Rate": 4.000000000000001e-06, "Finetune/Loss": 4.504611968994141, "Finetune/Loss (Raw)": 4.0129523277282715, "Finetune/Step": 4, "Finetune/Step Time": 18.859546840190887} +{"Finetune/Learning Rate": 5e-06, "Finetune/Loss": 4.199080467224121, "Finetune/Loss (Raw)": 2.9769539833068848, "Finetune/Step": 5, "Finetune/Step Time": 18.536297416687013} +{"Finetune/Learning Rate": 6e-06, "Finetune/Loss": 3.821427345275879, "Finetune/Loss (Raw)": 1.9331620931625366, "Finetune/Step": 6, "Finetune/Step Time": 18.240217328071594} +{"Finetune/Learning Rate": 7e-06, "Finetune/Loss": 3.512204647064209, "Finetune/Loss (Raw)": 1.6568677425384521, "Finetune/Step": 7, "Finetune/Step Time": 18.03140558515276} +{"Finetune/Learning Rate": 8.000000000000001e-06, "Finetune/Loss": 3.193592071533203, "Finetune/Loss (Raw)": 0.9633041620254517, "Finetune/Step": 8, "Finetune/Step Time": 17.913622170686722} +{"Finetune/Learning Rate": 9e-06, "Finetune/Loss": 2.923570156097412, "Finetune/Loss (Raw)": 0.7633946537971497, "Finetune/Step": 9, "Finetune/Step Time": 17.803975979487102} +{"Finetune/Learning Rate": 1e-05, "Finetune/Loss": 2.6761012077331543, "Finetune/Loss (Raw)": 0.4488799571990967, "Finetune/Step": 10, "Finetune/Step Time": 17.71381664276123} +{"Finetune/Learning Rate": 1.1000000000000001e-05, "Finetune/Loss": 2.4679646492004395, "Finetune/Loss (Raw)": 0.3866002559661865, "Finetune/Step": 11, "Finetune/Step Time": 17.63954916867343} +{"Finetune/Learning Rate": 1.2e-05, "Finetune/Loss": 2.2933290004730225, "Finetune/Loss (Raw)": 0.3723345398902893, "Finetune/Step": 12, "Finetune/Step Time": 17.577088197072346} +{"Finetune/Learning Rate": 1.3000000000000001e-05, "Finetune/Loss": 2.1396443843841553, "Finetune/Loss (Raw)": 0.2954293191432953, "Finetune/Step": 13, "Finetune/Step Time": 17.53974852195153} +{"Finetune/Learning Rate": 1.4e-05, "Finetune/Loss": 2.006793975830078, "Finetune/Loss (Raw)": 0.27973830699920654, "Finetune/Step": 14, "Finetune/Step Time": 17.54676912512098} +{"Finetune/Learning Rate": 1.5000000000000002e-05, "Finetune/Loss": 1.8925576210021973, "Finetune/Loss (Raw)": 0.29324865341186523, "Finetune/Step": 15, "Finetune/Step Time": 17.508335177103678} +{"Finetune/Learning Rate": 1.6000000000000003e-05, "Finetune/Loss": 1.7893580198287964, "Finetune/Loss (Raw)": 0.24136537313461304, "Finetune/Step": 16, "Finetune/Step Time": 17.555599838495255} +{"Finetune/Learning Rate": 1.7e-05, "Finetune/Loss": 1.6967027187347412, "Finetune/Loss (Raw)": 0.2142181098461151, "Finetune/Step": 17, "Finetune/Step Time": 17.58340079644147} +{"Finetune/Learning Rate": 1.8e-05, "Finetune/Loss": 1.6145164966583252, "Finetune/Loss (Raw)": 0.217349112033844, "Finetune/Step": 18, "Finetune/Step Time": 17.55183262295193} +{"Finetune/Learning Rate": 1.9e-05, "Finetune/Loss": 1.5386879444122314, "Finetune/Loss (Raw)": 0.17377321422100067, "Finetune/Step": 19, "Finetune/Step Time": 17.529371311790065} +{"Finetune/Learning Rate": 2e-05, "Finetune/Loss": 1.4705151319503784, "Finetune/Loss (Raw)": 0.17523084580898285, "Finetune/Step": 20, "Finetune/Step Time": 17.505639946460725} +{"Finetune/Learning Rate": 1.999988940995767e-05, "Finetune/Loss": 1.4096629619598389, "Finetune/Loss (Raw)": 0.19262146949768066, "Finetune/Step": 21, "Finetune/Step Time": 17.479219845363072} +{"Finetune/Learning Rate": 1.9999557642276715e-05, "Finetune/Loss": 1.3542253971099854, "Finetune/Loss (Raw)": 0.19003590941429138, "Finetune/Step": 22, "Finetune/Step Time": 17.44704456762834} +{"Finetune/Learning Rate": 1.9999004704295167e-05, "Finetune/Loss": 1.3026459217071533, "Finetune/Loss (Raw)": 0.16789904236793518, "Finetune/Step": 23, "Finetune/Step Time": 17.42296780710635} +{"Finetune/Learning Rate": 1.9998230608242917e-05, "Finetune/Loss": 1.2552615404129028, "Finetune/Loss (Raw)": 0.16541999578475952, "Finetune/Step": 24, "Finetune/Step Time": 17.464924196402233} +{"Finetune/Learning Rate": 1.999723537124143e-05, "Finetune/Loss": 1.211242914199829, "Finetune/Loss (Raw)": 0.1547965109348297, "Finetune/Step": 25, "Finetune/Step Time": 17.447522888183595} +{"Finetune/Learning Rate": 1.9996019015303366e-05, "Finetune/Loss": 1.1712974309921265, "Finetune/Loss (Raw)": 0.17266204953193665, "Finetune/Step": 26, "Finetune/Step Time": 17.45837808572329} +{"Finetune/Learning Rate": 1.9994581567332095e-05, "Finetune/Loss": 1.133945345878601, "Finetune/Loss (Raw)": 0.1627887785434723, "Finetune/Step": 27, "Finetune/Step Time": 17.451547622680664} +{"Finetune/Learning Rate": 1.9992923059121107e-05, "Finetune/Loss": 1.0987920761108398, "Finetune/Loss (Raw)": 0.1496562510728836, "Finetune/Step": 28, "Finetune/Step Time": 17.451610241617477} +{"Finetune/Learning Rate": 1.999104352735329e-05, "Finetune/Loss": 1.0661418437957764, "Finetune/Loss (Raw)": 0.15193572640419006, "Finetune/Step": 29, "Finetune/Step Time": 17.438449333454} +{"Finetune/Learning Rate": 1.9988943013600156e-05, "Finetune/Loss": 1.0352579355239868, "Finetune/Loss (Raw)": 0.139622762799263, "Finetune/Step": 30, "Finetune/Step Time": 17.431355158487957} +{"Finetune/Learning Rate": 1.9986621564320878e-05, "Finetune/Loss": 1.0072033405303955, "Finetune/Loss (Raw)": 0.16556519269943237, "Finetune/Step": 31, "Finetune/Step Time": 17.417596947762274} +{"Finetune/Learning Rate": 1.9984079230861296e-05, "Finetune/Loss": 0.98041832447052, "Finetune/Loss (Raw)": 0.15008476376533508, "Finetune/Step": 32, "Finetune/Step Time": 17.39851463586092} +{"Finetune/Learning Rate": 1.998131606945276e-05, "Finetune/Loss": 0.9554582238197327, "Finetune/Loss (Raw)": 0.15673428773880005, "Finetune/Step": 33, "Finetune/Step Time": 17.383506738778316} +{"Finetune/Learning Rate": 1.9978332141210894e-05, "Finetune/Loss": 0.9311700463294983, "Finetune/Loss (Raw)": 0.1296607106924057, "Finetune/Step": 34, "Finetune/Step Time": 17.371068239212036} +{"Finetune/Learning Rate": 1.997512751213426e-05, "Finetune/Loss": 0.9084528088569641, "Finetune/Loss (Raw)": 0.1360662430524826, "Finetune/Step": 35, "Finetune/Step Time": 17.383470364979335} +{"Finetune/Learning Rate": 1.9971702253102856e-05, "Finetune/Loss": 0.8863706588745117, "Finetune/Loss (Raw)": 0.11349590122699738, "Finetune/Step": 36, "Finetune/Step Time": 17.373878757158916} +{"Finetune/Learning Rate": 1.99680564398766e-05, "Finetune/Loss": 0.8661229014396667, "Finetune/Loss (Raw)": 0.13720020651817322, "Finetune/Step": 37, "Finetune/Step Time": 17.36750834052627} +{"Finetune/Learning Rate": 1.9964190153093613e-05, "Finetune/Loss": 0.8469014763832092, "Finetune/Loss (Raw)": 0.13570785522460938, "Finetune/Step": 38, "Finetune/Step Time": 17.390725317754242} +{"Finetune/Learning Rate": 1.996010347826847e-05, "Finetune/Loss": 0.8285725116729736, "Finetune/Loss (Raw)": 0.1320754885673523, "Finetune/Step": 39, "Finetune/Step Time": 17.39677649889237} +{"Finetune/Learning Rate": 1.9955796505790266e-05, "Finetune/Loss": 0.8112174272537231, "Finetune/Loss (Raw)": 0.1343693882226944, "Finetune/Step": 40, "Finetune/Step Time": 17.400323623418807} +{"Finetune/Learning Rate": 1.9951269330920667e-05, "Finetune/Loss": 0.7941542267799377, "Finetune/Loss (Raw)": 0.11163019388914108, "Finetune/Step": 41, "Finetune/Step Time": 17.387319029831303} +{"Finetune/Learning Rate": 1.9946522053791755e-05, "Finetune/Loss": 0.7779905200004578, "Finetune/Loss (Raw)": 0.11527504026889801, "Finetune/Step": 42, "Finetune/Step Time": 17.38388817083268} +{"Finetune/Learning Rate": 1.9941554779403854e-05, "Finetune/Loss": 0.7627388834953308, "Finetune/Loss (Raw)": 0.12217073142528534, "Finetune/Step": 43, "Finetune/Step Time": 17.381137531857156} +{"Finetune/Learning Rate": 1.993636761762317e-05, "Finetune/Loss": 0.7480058073997498, "Finetune/Loss (Raw)": 0.11447931826114655, "Finetune/Step": 44, "Finetune/Step Time": 17.377234784039583} +{"Finetune/Learning Rate": 1.9930960683179404e-05, "Finetune/Loss": 0.7339156270027161, "Finetune/Loss (Raw)": 0.11395035684108734, "Finetune/Step": 45, "Finetune/Step Time": 17.38643084632026} +{"Finetune/Learning Rate": 1.992533409566317e-05, "Finetune/Loss": 0.7209261059761047, "Finetune/Loss (Raw)": 0.13639730215072632, "Finetune/Step": 46, "Finetune/Step Time": 17.39419576396113} +{"Finetune/Learning Rate": 1.991948797952338e-05, "Finetune/Loss": 0.7084022164344788, "Finetune/Loss (Raw)": 0.13230812549591064, "Finetune/Step": 47, "Finetune/Step Time": 17.379062469969405} +{"Finetune/Learning Rate": 1.991342246406448e-05, "Finetune/Loss": 0.6961833834648132, "Finetune/Loss (Raw)": 0.12189091742038727, "Finetune/Step": 48, "Finetune/Step Time": 17.382669900854427} +{"Finetune/Learning Rate": 1.9907137683443593e-05, "Finetune/Loss": 0.6841696500778198, "Finetune/Loss (Raw)": 0.1075170561671257, "Finetune/Step": 49, "Finetune/Step Time": 17.40413404970753} +{"Finetune/Learning Rate": 1.990063377666755e-05, "Finetune/Loss": 0.6728960871696472, "Finetune/Loss (Raw)": 0.1204880028963089, "Finetune/Step": 50, "Finetune/Step Time": 17.40959596633911} +{"Finetune/Learning Rate": 1.9893910887589817e-05, "Finetune/Loss": 0.6617352366447449, "Finetune/Loss (Raw)": 0.10369187593460083, "Finetune/Step": 51, "Finetune/Step Time": 17.401254476285448} +{"Finetune/Learning Rate": 1.9886969164907306e-05, "Finetune/Loss": 0.6512115597724915, "Finetune/Loss (Raw)": 0.11450354009866714, "Finetune/Step": 52, "Finetune/Step Time": 17.412318619397972} +{"Finetune/Learning Rate": 1.9879808762157106e-05, "Finetune/Loss": 0.6411347389221191, "Finetune/Loss (Raw)": 0.1171388328075409, "Finetune/Step": 53, "Finetune/Step Time": 17.40480715373777} +{"Finetune/Learning Rate": 1.987242983771306e-05, "Finetune/Loss": 0.6314305663108826, "Finetune/Loss (Raw)": 0.11711299419403076, "Finetune/Step": 54, "Finetune/Step Time": 17.398192630873787} +{"Finetune/Learning Rate": 1.986483255478229e-05, "Finetune/Loss": 0.6216724514961243, "Finetune/Loss (Raw)": 0.09473195672035217, "Finetune/Step": 55, "Finetune/Step Time": 17.420223569869997} +{"Finetune/Learning Rate": 1.985701708140155e-05, "Finetune/Loss": 0.6123361587524414, "Finetune/Loss (Raw)": 0.0988379567861557, "Finetune/Step": 56, "Finetune/Step Time": 17.414544343948364} +{"Finetune/Learning Rate": 1.0000000000000002e-06, "Finetune/Loss": 4.717222213745117, "Finetune/Loss (Raw)": 4.717222213745117, "Finetune/Step": 1, "Finetune/Step Time": 25.17544674873352} +{"Finetune/Learning Rate": 2.0000000000000003e-06, "Finetune/Loss": 4.7392425537109375, "Finetune/Loss (Raw)": 4.761262893676758, "Finetune/Step": 2, "Finetune/Step Time": 22.209756016731262} +{"Finetune/Learning Rate": 3e-06, "Finetune/Loss": 4.683249473571777, "Finetune/Loss (Raw)": 4.571263790130615, "Finetune/Step": 3, "Finetune/Step Time": 20.895675261815388} +{"Finetune/Learning Rate": 4.000000000000001e-06, "Finetune/Loss": 4.508266448974609, "Finetune/Loss (Raw)": 3.983315944671631, "Finetune/Step": 4, "Finetune/Step Time": 20.084836900234222} +{"Finetune/Learning Rate": 5e-06, "Finetune/Loss": 4.197574138641357, "Finetune/Loss (Raw)": 2.954803943634033, "Finetune/Step": 5, "Finetune/Step Time": 19.526369428634645} +{"Finetune/Learning Rate": 1.0000000000000002e-06, "Finetune/Loss": 4.717222213745117, "Finetune/Loss (Raw)": 4.717222213745117, "Finetune/Step": 1, "Finetune/Step Time": 22.019757509231567} +{"Finetune/Learning Rate": 2.0000000000000003e-06, "Finetune/Loss": 4.7392425537109375, "Finetune/Loss (Raw)": 4.761262893676758, "Finetune/Step": 2, "Finetune/Step Time": 20.48360002040863} +{"Finetune/Learning Rate": 3e-06, "Finetune/Loss": 4.682786464691162, "Finetune/Loss (Raw)": 4.569873809814453, "Finetune/Step": 3, "Finetune/Step Time": 19.70590392748515} +{"Finetune/Learning Rate": 4.000000000000001e-06, "Finetune/Loss": 4.507586479187012, "Finetune/Loss (Raw)": 3.9819865226745605, "Finetune/Step": 4, "Finetune/Step Time": 19.18842512369156} +{"Finetune/Learning Rate": 5e-06, "Finetune/Loss": 4.197033882141113, "Finetune/Loss (Raw)": 2.9548234939575195, "Finetune/Step": 5, "Finetune/Step Time": 18.80903067588806} +{"Finetune/Learning Rate": 6e-06, "Finetune/Loss": 3.819854259490967, "Finetune/Loss (Raw)": 1.933955192565918, "Finetune/Step": 6, "Finetune/Step Time": 18.498749891916912} +{"Finetune/Learning Rate": 1.0000000000000002e-06, "Finetune/Loss": 4.695167064666748, "Finetune/Loss (Raw)": 4.695167064666748, "Finetune/Step": 1, "Finetune/Step Time": 24.950685739517212} +{"Finetune/Learning Rate": 1.0000000000000002e-06, "Finetune/Loss": 4.872750282287598, "Finetune/Loss (Raw)": 4.872750282287598, "Finetune/Step": 1, "Finetune/Step Time": 21.23921251296997} +{"Finetune/Learning Rate": 2.0000000000000003e-06, "Finetune/Loss": 4.861178874969482, "Finetune/Loss (Raw)": 4.849607467651367, "Finetune/Step": 2, "Finetune/Step Time": 20.036610007286072} +{"Finetune/Learning Rate": 3e-06, "Finetune/Loss": 4.802614212036133, "Finetune/Loss (Raw)": 4.68548583984375, "Finetune/Step": 3, "Finetune/Step Time": 19.36487563451131} +{"Finetune/Learning Rate": 4.000000000000001e-06, "Finetune/Loss": 4.6239118576049805, "Finetune/Loss (Raw)": 4.087802410125732, "Finetune/Step": 4, "Finetune/Step Time": 18.85390079021454} +{"Finetune/Learning Rate": 5e-06, "Finetune/Loss": 4.323788166046143, "Finetune/Loss (Raw)": 3.123293876647949, "Finetune/Step": 5, "Finetune/Step Time": 18.518577003479002} +{"Finetune/Learning Rate": 6e-06, "Finetune/Loss": 3.934563636779785, "Finetune/Loss (Raw)": 1.9884411096572876, "Finetune/Step": 6, "Finetune/Step Time": 18.188905239105225} +{"Finetune/Learning Rate": 7e-06, "Finetune/Loss": 3.6209516525268555, "Finetune/Loss (Raw)": 1.7392784357070923, "Finetune/Step": 7, "Finetune/Step Time": 17.968812465667725} +{"Finetune/Learning Rate": 8.000000000000001e-06, "Finetune/Loss": 3.2825136184692383, "Finetune/Loss (Raw)": 0.9134484529495239, "Finetune/Step": 8, "Finetune/Step Time": 17.96453520655632} +{"Finetune/Learning Rate": 9e-06, "Finetune/Loss": 2.998053789138794, "Finetune/Loss (Raw)": 0.722377359867096, "Finetune/Step": 9, "Finetune/Step Time": 17.82899504237705} +{"Finetune/Learning Rate": 1e-05, "Finetune/Loss": 2.7457494735717773, "Finetune/Loss (Raw)": 0.4750087857246399, "Finetune/Step": 10, "Finetune/Step Time": 17.723174262046815} +{"Finetune/Learning Rate": 1.1000000000000001e-05, "Finetune/Loss": 2.5370266437530518, "Finetune/Loss (Raw)": 0.4497969150543213, "Finetune/Step": 11, "Finetune/Step Time": 17.636180357499555} +{"Finetune/Learning Rate": 1.2e-05, "Finetune/Loss": 2.3582866191864014, "Finetune/Loss (Raw)": 0.39214685559272766, "Finetune/Step": 12, "Finetune/Step Time": 17.55503636598587} +{"Finetune/Learning Rate": 1.3000000000000001e-05, "Finetune/Loss": 2.2035064697265625, "Finetune/Loss (Raw)": 0.34614676237106323, "Finetune/Step": 13, "Finetune/Step Time": 17.492802198116596} +{"Finetune/Learning Rate": 1.4e-05, "Finetune/Loss": 2.067488193511963, "Finetune/Loss (Raw)": 0.299248605966568, "Finetune/Step": 14, "Finetune/Step Time": 17.545819180352346} +{"Finetune/Learning Rate": 1.5000000000000002e-05, "Finetune/Loss": 1.9474979639053345, "Finetune/Loss (Raw)": 0.26763206720352173, "Finetune/Step": 15, "Finetune/Step Time": 17.49828011194865} +{"Finetune/Learning Rate": 1.6000000000000003e-05, "Finetune/Loss": 1.8416781425476074, "Finetune/Loss (Raw)": 0.2543836534023285, "Finetune/Step": 16, "Finetune/Step Time": 17.548593625426292} +{"Finetune/Learning Rate": 1.7e-05, "Finetune/Loss": 1.7466574907302856, "Finetune/Loss (Raw)": 0.2263268381357193, "Finetune/Step": 17, "Finetune/Step Time": 17.55972427480361} +{"Finetune/Learning Rate": 1.8e-05, "Finetune/Loss": 1.6614431142807007, "Finetune/Loss (Raw)": 0.2127997875213623, "Finetune/Step": 18, "Finetune/Step Time": 17.527890920639038} +{"Finetune/Learning Rate": 1.9e-05, "Finetune/Loss": 1.5851675271987915, "Finetune/Loss (Raw)": 0.21220830082893372, "Finetune/Step": 19, "Finetune/Step Time": 17.495089970136945} +{"Finetune/Learning Rate": 2e-05, "Finetune/Loss": 1.5152664184570312, "Finetune/Loss (Raw)": 0.18714579939842224, "Finetune/Step": 20, "Finetune/Step Time": 17.461451995372773} +{"Finetune/Learning Rate": 1.999988940995767e-05, "Finetune/Loss": 1.4527583122253418, "Finetune/Loss (Raw)": 0.20259548723697662, "Finetune/Step": 21, "Finetune/Step Time": 17.429166260219755} +{"Finetune/Learning Rate": 1.9999557642276715e-05, "Finetune/Loss": 1.3957033157348633, "Finetune/Loss (Raw)": 0.19754715263843536, "Finetune/Step": 22, "Finetune/Step Time": 17.38408304344524} +{"Finetune/Learning Rate": 1.9999004704295167e-05, "Finetune/Loss": 1.3420722484588623, "Finetune/Loss (Raw)": 0.16219186782836914, "Finetune/Step": 23, "Finetune/Step Time": 17.355615657308828} +{"Finetune/Learning Rate": 1.9998230608242917e-05, "Finetune/Loss": 1.293485164642334, "Finetune/Loss (Raw)": 0.1759803593158722, "Finetune/Step": 24, "Finetune/Step Time": 17.412310471137364} +{"Finetune/Learning Rate": 1.999723537124143e-05, "Finetune/Loss": 1.2485833168029785, "Finetune/Loss (Raw)": 0.17093876004219055, "Finetune/Step": 25, "Finetune/Step Time": 17.390757474899292} diff --git a/run-metrics.jsonl b/run-metrics.jsonl new file mode 100644 index 0000000000000000000000000000000000000000..4f7ce907b29b9343ec480e3999297ccf628e39f0 --- /dev/null +++ b/run-metrics.jsonl @@ -0,0 +1 @@ +{"hparams": {"dataset": {"align_stage_components": ["download/med-v0.1-instruct/med_v0_1_mix.json", "download/med-v0.1-instruct"], "dataset_id": "med-instruct", "dataset_root_dir": "data", "finetune_stage_components": ["download/med-v0.1-instruct/med_v0_1_mix.json", "download/med-v0.1-instruct"], "type": "med-instruct"}, "model": {"align_epochs": 1, "align_global_batch_size": 256, "align_learning_rate": 0.001, "align_lr_scheduler_type": "linear-warmup+cosine-decay", "align_max_grad_norm": 1.0, "align_max_steps": null, "align_per_device_batch_size": 16, "align_train_strategy": "fsdp-shard-grad-op", "align_warmup_ratio": 0.03, "align_weight_decay": 0.0, "arch_specifier": "no-align+gelu-mlp", "enable_gradient_checkpointing": true, "enable_mixed_precision_training": true, "finetune_epochs": 1, "finetune_global_batch_size": 128, "finetune_learning_rate": 2e-05, "finetune_lr_scheduler_type": "linear-warmup+cosine-decay", "finetune_max_grad_norm": 1.0, "finetune_max_steps": null, "finetune_per_device_batch_size": 16, "finetune_train_strategy": "fsdp-full-shard", "finetune_warmup_ratio": 0.03, "finetune_weight_decay": 0.1, "image_resize_strategy": "letterbox", "llm_backbone_id": "llama2-7b-chat", "llm_max_length": 2048, "model_id": "testmodel", "reduce_in_full_precision": false, "type": "one-stage+7b", "vision_backbone_id": "clip-vit-b"}, "pretrained_checkpoint": null, "run_id": "med-instruct+testmodel+stage-finetune+x7", "run_root_dir": "runs", "seed": 7, "stage": "finetune", "trackers": ["jsonl", "wandb"], "wandb_entity": "stanford-voltron", "wandb_project": "onyx-vlms"}, "run_id": "med-instruct+testmodel+stage-finetune+x7"} diff --git a/wandb/debug-internal.log b/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..de9b3fe2a2a3fb9dfd5c9b35160940acf01b9258 --- /dev/null +++ b/wandb/debug-internal.log @@ -0,0 +1,337 @@ +2024-05-09 14:01:23,076 INFO StreamThr :4861 [internal.py:wandb_internal():86] W&B internal server running at pid: 4861, started at: 2024-05-09 14:01:23.074424 +2024-05-09 14:01:23,078 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status +2024-05-09 14:01:23,086 INFO WriterThread:4861 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/run-vqw25nkx.wandb +2024-05-09 14:01:23,143 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: run_start +2024-05-09 14:01:23,165 DEBUG HandlerThread:4861 [system_info.py:__init__():26] System info init +2024-05-09 14:01:23,165 DEBUG HandlerThread:4861 [system_info.py:__init__():41] System info init done +2024-05-09 14:01:23,165 INFO HandlerThread:4861 [system_monitor.py:start():194] Starting system monitor +2024-05-09 14:01:23,165 INFO SystemMonitor:4861 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-09 14:01:23,165 INFO HandlerThread:4861 [system_monitor.py:probe():214] Collecting system info +2024-05-09 14:01:23,167 INFO SystemMonitor:4861 [interfaces.py:start():190] Started cpu monitoring +2024-05-09 14:01:23,168 INFO SystemMonitor:4861 [interfaces.py:start():190] Started disk monitoring +2024-05-09 14:01:23,169 INFO SystemMonitor:4861 [interfaces.py:start():190] Started gpu monitoring +2024-05-09 14:01:23,170 INFO SystemMonitor:4861 [interfaces.py:start():190] Started memory monitoring +2024-05-09 14:01:23,170 INFO SystemMonitor:4861 [interfaces.py:start():190] Started network monitoring +2024-05-09 14:01:23,197 DEBUG HandlerThread:4861 [system_info.py:probe():150] Probing system +2024-05-09 14:01:23,199 DEBUG HandlerThread:4861 [system_info.py:_probe_git():135] Probing git +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_info.py:_probe_git():143] Probing git done +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_info.py:probe():198] Probing system done +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T21:01:23.197793', 'startedAt': '2024-05-09T21:01:23.056501', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '128', '--model.finetune_per_device_batch_size', '16', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-11n01.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.9468749999999992, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.626476287841797}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5838279724121}} +2024-05-09 14:01:23,209 INFO HandlerThread:4861 [system_monitor.py:probe():224] Finished collecting system info +2024-05-09 14:01:23,209 INFO HandlerThread:4861 [system_monitor.py:probe():227] Publishing system info +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-09 14:01:30,038 DEBUG HandlerThread:4861 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-09 14:01:30,042 INFO HandlerThread:4861 [system_monitor.py:probe():229] Finished publishing system info +2024-05-09 14:01:30,047 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:30,048 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: keepalive +2024-05-09 14:01:30,048 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:30,270 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: python_packages +2024-05-09 14:01:34,049 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:34,050 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:39,051 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:39,051 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:44,052 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:44,052 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:49,053 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:49,054 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:51,803 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:01:54,055 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:54,055 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:59,056 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:59,056 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:04,057 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:04,057 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:09,058 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:09,059 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:10,625 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:02:14,060 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:14,060 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:19,061 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:19,061 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:23,171 DEBUG SystemMonitor:4861 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-09 14:02:24,062 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:24,062 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:28,604 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:02:29,063 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:29,063 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:34,064 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:34,065 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:39,065 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:39,066 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:44,066 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:44,067 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:45,970 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:02:49,068 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:49,068 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:54,069 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:54,069 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:59,070 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:59,071 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:03,109 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:04,072 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:04,072 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:09,073 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:09,073 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:14,074 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:14,075 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:19,075 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:19,076 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:19,636 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:24,077 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:24,077 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:29,078 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:29,078 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:34,079 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:34,079 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:36,299 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:39,080 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:39,081 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:44,082 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:44,082 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:49,083 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:49,083 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:54,084 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:54,085 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:54,227 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:59,085 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:59,086 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:04,087 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:04,087 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:09,088 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:09,088 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:10,990 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:04:14,089 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:14,089 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:19,090 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:19,091 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:24,092 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:24,092 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:27,771 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:04:29,093 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:29,094 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:34,095 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:34,095 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:39,096 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:39,096 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:44,097 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:44,098 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:44,534 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:04:49,098 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:49,099 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:54,100 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:54,100 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:59,101 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:59,101 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:01,175 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:04,102 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:04,103 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:09,103 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:09,104 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:14,105 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:14,105 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:17,930 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:19,106 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:19,106 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:24,107 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:24,107 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:29,108 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:29,108 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:34,109 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:34,110 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:36,171 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:39,111 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:39,111 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:44,112 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:44,112 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:49,113 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:49,113 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:52,984 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:54,114 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:54,115 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:59,115 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:59,116 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:04,117 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:04,117 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:09,118 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:09,118 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:11,350 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:06:14,119 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:14,119 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:19,120 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:19,120 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:24,121 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:24,122 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:29,068 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:06:29,122 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:29,123 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:34,123 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:34,124 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:39,125 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:39,125 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:44,126 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:44,126 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:46,015 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:06:49,127 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:49,128 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:54,129 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:54,129 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:59,130 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:59,130 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:02,938 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:04,131 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:04,131 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:09,132 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:09,132 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:14,133 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:14,134 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:19,134 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:19,135 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:19,747 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:24,135 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:24,136 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:29,137 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:29,137 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:34,138 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:34,138 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:36,532 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:39,139 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:39,140 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:44,141 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:44,141 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:49,142 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:49,142 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:52,968 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:54,143 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:54,144 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:59,144 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:59,145 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:04,146 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:04,146 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:09,147 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:09,147 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:09,692 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:08:14,148 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:14,149 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:19,149 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:19,150 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:24,150 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:24,151 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:28,467 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:08:29,152 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:29,152 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:34,153 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:34,153 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:39,154 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:39,154 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:44,155 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:44,156 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:45,278 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:08:49,157 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:49,157 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:54,158 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:54,159 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:55,939 DEBUG SenderThread:4861 [sender.py:send():379] send: exit +2024-05-09 14:08:55,939 INFO SenderThread:4861 [sender.py:send_exit():586] handling exit code: 255 +2024-05-09 14:08:55,939 INFO SenderThread:4861 [sender.py:send_exit():588] handling runtime: 452 +2024-05-09 14:08:55,943 INFO SenderThread:4861 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-09 14:08:55,943 INFO SenderThread:4861 [sender.py:send_exit():594] send defer +2024-05-09 14:08:55,944 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,944 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 0 +2024-05-09 14:08:55,944 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,944 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-05-09 14:08:55,944 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 1 +2024-05-09 14:08:55,944 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,944 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 1 +2024-05-09 14:08:55,945 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,945 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-05-09 14:08:55,945 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 2 +2024-05-09 14:08:55,945 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,945 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 2 +2024-05-09 14:08:55,945 INFO HandlerThread:4861 [system_monitor.py:finish():203] Stopping system monitor +2024-05-09 14:08:55,946 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined cpu monitor +2024-05-09 14:08:55,946 DEBUG SystemMonitor:4861 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-09 14:08:55,946 DEBUG SystemMonitor:4861 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-09 14:08:55,954 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined disk monitor +2024-05-09 14:08:55,971 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined gpu monitor +2024-05-09 14:08:55,971 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined memory monitor +2024-05-09 14:08:55,971 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined network monitor +2024-05-09 14:08:55,972 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,972 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-05-09 14:08:55,972 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 3 +2024-05-09 14:08:55,972 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,972 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 3 +2024-05-09 14:08:55,973 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,973 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-05-09 14:08:55,973 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 4 +2024-05-09 14:08:55,973 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,973 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 4 +2024-05-09 14:08:55,974 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 5 +2024-05-09 14:08:55,974 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,974 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 5 +2024-05-09 14:08:55,974 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 6 +2024-05-09 14:08:55,974 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,975 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 6 +2024-05-09 14:08:55,975 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 7 +2024-05-09 14:08:55,975 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:55,975 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,975 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 7 +2024-05-09 14:08:55,975 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:55,975 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 8 +2024-05-09 14:08:55,975 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 8 +2024-05-09 14:08:55,976 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 9 +2024-05-09 14:08:55,976 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 9 +2024-05-09 14:08:55,976 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 10 +2024-05-09 14:08:55,976 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 10 +2024-05-09 14:08:55,976 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 11 +2024-05-09 14:08:55,976 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 11 +2024-05-09 14:08:55,977 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 12 +2024-05-09 14:08:55,977 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,977 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 12 +2024-05-09 14:08:55,977 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 13 +2024-05-09 14:08:55,977 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,977 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 13 +2024-05-09 14:08:55,977 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 14 +2024-05-09 14:08:55,977 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,978 DEBUG SenderThread:4861 [sender.py:send():379] send: final +2024-05-09 14:08:55,978 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 14 +2024-05-09 14:08:55,978 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,978 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-05-09 14:08:55,978 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-09 14:08:55,979 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: poll_exit +2024-05-09 14:08:55,979 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-09 14:08:55,979 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: poll_exit +2024-05-09 14:08:55,979 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: server_info +2024-05-09 14:08:55,980 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: server_info +2024-05-09 14:08:55,980 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: get_summary +2024-05-09 14:08:55,980 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: sampled_history +2024-05-09 14:08:55,981 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-09 14:08:55,981 INFO MainThread:4861 [wandb_run.py:_footer_history_summary_info():3936] rendering history +2024-05-09 14:08:55,982 INFO MainThread:4861 [wandb_run.py:_footer_history_summary_info():3968] rendering summary +2024-05-09 14:08:55,983 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: shutdown +2024-05-09 14:08:55,983 INFO HandlerThread:4861 [handler.py:finish():866] shutting down handler +2024-05-09 14:08:56,980 INFO WriterThread:4861 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/run-vqw25nkx.wandb +2024-05-09 14:08:56,980 INFO SenderThread:4861 [sender.py:finish():1546] shutting down sender diff --git a/wandb/debug.log b/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..20f16f66df3d908b0be35c3bfcb0701e7bc181c6 --- /dev/null +++ b/wandb/debug.log @@ -0,0 +1,27 @@ +2024-05-09 14:01:23,067 INFO MainThread:4587 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Configure stats pid to 4587 +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug.log +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug-internal.log +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():561] calling init triggers +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():611] starting backend +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():615] setting up manager +2024-05-09 14:01:23,073 INFO MainThread:4587 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-09 14:01:23,075 INFO MainThread:4587 [wandb_init.py:init():623] backend started and connected +2024-05-09 14:01:23,086 INFO MainThread:4587 [wandb_init.py:init():715] updated telemetry +2024-05-09 14:01:23,094 INFO MainThread:4587 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-09 14:01:23,099 INFO MainThread:4587 [wandb_init.py:init():799] starting run threads in backend +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_console_start():2335] atexit reg +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-09 14:01:30,271 INFO MainThread:4587 [wandb_init.py:init():842] run started, returning control to user process +2024-05-09 14:08:56,984 WARNING MsgRouterThr:4587 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/latest-run/files/conda-environment.yaml b/wandb/latest-run/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb5c46cd8a524560371e6601fcdbf9ef63efe33a --- /dev/null +++ b/wandb/latest-run/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.40.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/latest-run/files/wandb-metadata.json b/wandb/latest-run/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b5a769117a47be551e84c8e4dcba4e1b21bb1051 --- /dev/null +++ b/wandb/latest-run/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T21:01:23.197793", + "startedAt": "2024-05-09T21:01:23.056501", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "128", + "--model.finetune_per_device_batch_size", + "16", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-11n01.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.9468749999999992, + "min": 1500.0, + "max": 2800.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.626476287841797 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5838279724121 + } +} diff --git a/wandb/latest-run/files/wandb-summary.json b/wandb/latest-run/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c705b16012d5dda581b1f3ccbe2518fd9c91ae53 --- /dev/null +++ b/wandb/latest-run/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 452}} \ No newline at end of file diff --git a/wandb/latest-run/logs/debug-internal.log b/wandb/latest-run/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..de9b3fe2a2a3fb9dfd5c9b35160940acf01b9258 --- /dev/null +++ b/wandb/latest-run/logs/debug-internal.log @@ -0,0 +1,337 @@ +2024-05-09 14:01:23,076 INFO StreamThr :4861 [internal.py:wandb_internal():86] W&B internal server running at pid: 4861, started at: 2024-05-09 14:01:23.074424 +2024-05-09 14:01:23,078 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status +2024-05-09 14:01:23,086 INFO WriterThread:4861 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/run-vqw25nkx.wandb +2024-05-09 14:01:23,143 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: run_start +2024-05-09 14:01:23,165 DEBUG HandlerThread:4861 [system_info.py:__init__():26] System info init +2024-05-09 14:01:23,165 DEBUG HandlerThread:4861 [system_info.py:__init__():41] System info init done +2024-05-09 14:01:23,165 INFO HandlerThread:4861 [system_monitor.py:start():194] Starting system monitor +2024-05-09 14:01:23,165 INFO SystemMonitor:4861 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-09 14:01:23,165 INFO HandlerThread:4861 [system_monitor.py:probe():214] Collecting system info +2024-05-09 14:01:23,167 INFO SystemMonitor:4861 [interfaces.py:start():190] Started cpu monitoring +2024-05-09 14:01:23,168 INFO SystemMonitor:4861 [interfaces.py:start():190] Started disk monitoring +2024-05-09 14:01:23,169 INFO SystemMonitor:4861 [interfaces.py:start():190] Started gpu monitoring +2024-05-09 14:01:23,170 INFO SystemMonitor:4861 [interfaces.py:start():190] Started memory monitoring +2024-05-09 14:01:23,170 INFO SystemMonitor:4861 [interfaces.py:start():190] Started network monitoring +2024-05-09 14:01:23,197 DEBUG HandlerThread:4861 [system_info.py:probe():150] Probing system +2024-05-09 14:01:23,199 DEBUG HandlerThread:4861 [system_info.py:_probe_git():135] Probing git +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_info.py:_probe_git():143] Probing git done +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_info.py:probe():198] Probing system done +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T21:01:23.197793', 'startedAt': '2024-05-09T21:01:23.056501', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '128', '--model.finetune_per_device_batch_size', '16', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-11n01.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.9468749999999992, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.626476287841797}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5838279724121}} +2024-05-09 14:01:23,209 INFO HandlerThread:4861 [system_monitor.py:probe():224] Finished collecting system info +2024-05-09 14:01:23,209 INFO HandlerThread:4861 [system_monitor.py:probe():227] Publishing system info +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-09 14:01:30,038 DEBUG HandlerThread:4861 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-09 14:01:30,042 INFO HandlerThread:4861 [system_monitor.py:probe():229] Finished publishing system info +2024-05-09 14:01:30,047 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:30,048 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: keepalive +2024-05-09 14:01:30,048 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:30,270 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: python_packages +2024-05-09 14:01:34,049 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:34,050 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:39,051 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:39,051 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:44,052 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:44,052 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:49,053 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:49,054 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:51,803 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:01:54,055 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:54,055 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:59,056 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:59,056 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:04,057 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:04,057 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:09,058 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:09,059 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:10,625 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:02:14,060 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:14,060 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:19,061 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:19,061 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:23,171 DEBUG SystemMonitor:4861 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-09 14:02:24,062 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:24,062 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:28,604 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:02:29,063 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:29,063 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:34,064 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:34,065 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:39,065 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:39,066 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:44,066 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:44,067 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:45,970 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:02:49,068 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:49,068 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:54,069 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:54,069 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:59,070 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:59,071 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:03,109 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:04,072 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:04,072 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:09,073 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:09,073 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:14,074 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:14,075 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:19,075 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:19,076 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:19,636 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:24,077 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:24,077 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:29,078 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:29,078 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:34,079 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:34,079 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:36,299 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:39,080 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:39,081 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:44,082 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:44,082 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:49,083 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:49,083 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:54,084 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:54,085 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:54,227 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:59,085 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:59,086 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:04,087 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:04,087 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:09,088 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:09,088 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:10,990 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:04:14,089 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:14,089 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:19,090 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:19,091 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:24,092 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:24,092 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:27,771 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:04:29,093 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:29,094 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:34,095 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:34,095 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:39,096 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:39,096 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:44,097 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:44,098 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:44,534 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:04:49,098 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:49,099 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:54,100 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:54,100 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:59,101 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:59,101 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:01,175 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:04,102 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:04,103 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:09,103 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:09,104 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:14,105 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:14,105 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:17,930 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:19,106 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:19,106 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:24,107 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:24,107 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:29,108 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:29,108 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:34,109 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:34,110 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:36,171 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:39,111 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:39,111 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:44,112 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:44,112 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:49,113 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:49,113 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:52,984 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:54,114 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:54,115 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:59,115 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:59,116 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:04,117 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:04,117 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:09,118 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:09,118 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:11,350 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:06:14,119 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:14,119 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:19,120 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:19,120 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:24,121 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:24,122 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:29,068 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:06:29,122 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:29,123 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:34,123 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:34,124 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:39,125 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:39,125 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:44,126 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:44,126 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:46,015 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:06:49,127 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:49,128 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:54,129 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:54,129 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:59,130 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:59,130 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:02,938 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:04,131 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:04,131 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:09,132 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:09,132 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:14,133 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:14,134 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:19,134 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:19,135 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:19,747 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:24,135 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:24,136 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:29,137 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:29,137 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:34,138 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:34,138 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:36,532 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:39,139 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:39,140 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:44,141 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:44,141 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:49,142 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:49,142 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:52,968 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:54,143 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:54,144 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:59,144 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:59,145 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:04,146 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:04,146 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:09,147 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:09,147 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:09,692 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:08:14,148 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:14,149 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:19,149 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:19,150 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:24,150 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:24,151 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:28,467 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:08:29,152 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:29,152 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:34,153 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:34,153 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:39,154 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:39,154 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:44,155 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:44,156 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:45,278 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:08:49,157 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:49,157 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:54,158 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:54,159 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:55,939 DEBUG SenderThread:4861 [sender.py:send():379] send: exit +2024-05-09 14:08:55,939 INFO SenderThread:4861 [sender.py:send_exit():586] handling exit code: 255 +2024-05-09 14:08:55,939 INFO SenderThread:4861 [sender.py:send_exit():588] handling runtime: 452 +2024-05-09 14:08:55,943 INFO SenderThread:4861 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-09 14:08:55,943 INFO SenderThread:4861 [sender.py:send_exit():594] send defer +2024-05-09 14:08:55,944 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,944 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 0 +2024-05-09 14:08:55,944 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,944 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-05-09 14:08:55,944 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 1 +2024-05-09 14:08:55,944 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,944 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 1 +2024-05-09 14:08:55,945 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,945 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-05-09 14:08:55,945 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 2 +2024-05-09 14:08:55,945 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,945 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 2 +2024-05-09 14:08:55,945 INFO HandlerThread:4861 [system_monitor.py:finish():203] Stopping system monitor +2024-05-09 14:08:55,946 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined cpu monitor +2024-05-09 14:08:55,946 DEBUG SystemMonitor:4861 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-09 14:08:55,946 DEBUG SystemMonitor:4861 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-09 14:08:55,954 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined disk monitor +2024-05-09 14:08:55,971 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined gpu monitor +2024-05-09 14:08:55,971 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined memory monitor +2024-05-09 14:08:55,971 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined network monitor +2024-05-09 14:08:55,972 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,972 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-05-09 14:08:55,972 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 3 +2024-05-09 14:08:55,972 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,972 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 3 +2024-05-09 14:08:55,973 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,973 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-05-09 14:08:55,973 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 4 +2024-05-09 14:08:55,973 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,973 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 4 +2024-05-09 14:08:55,974 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 5 +2024-05-09 14:08:55,974 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,974 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 5 +2024-05-09 14:08:55,974 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 6 +2024-05-09 14:08:55,974 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,975 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 6 +2024-05-09 14:08:55,975 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 7 +2024-05-09 14:08:55,975 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:55,975 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,975 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 7 +2024-05-09 14:08:55,975 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:55,975 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 8 +2024-05-09 14:08:55,975 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 8 +2024-05-09 14:08:55,976 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 9 +2024-05-09 14:08:55,976 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 9 +2024-05-09 14:08:55,976 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 10 +2024-05-09 14:08:55,976 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 10 +2024-05-09 14:08:55,976 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 11 +2024-05-09 14:08:55,976 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 11 +2024-05-09 14:08:55,977 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 12 +2024-05-09 14:08:55,977 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,977 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 12 +2024-05-09 14:08:55,977 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 13 +2024-05-09 14:08:55,977 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,977 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 13 +2024-05-09 14:08:55,977 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 14 +2024-05-09 14:08:55,977 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,978 DEBUG SenderThread:4861 [sender.py:send():379] send: final +2024-05-09 14:08:55,978 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 14 +2024-05-09 14:08:55,978 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,978 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-05-09 14:08:55,978 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-09 14:08:55,979 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: poll_exit +2024-05-09 14:08:55,979 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-09 14:08:55,979 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: poll_exit +2024-05-09 14:08:55,979 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: server_info +2024-05-09 14:08:55,980 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: server_info +2024-05-09 14:08:55,980 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: get_summary +2024-05-09 14:08:55,980 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: sampled_history +2024-05-09 14:08:55,981 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-09 14:08:55,981 INFO MainThread:4861 [wandb_run.py:_footer_history_summary_info():3936] rendering history +2024-05-09 14:08:55,982 INFO MainThread:4861 [wandb_run.py:_footer_history_summary_info():3968] rendering summary +2024-05-09 14:08:55,983 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: shutdown +2024-05-09 14:08:55,983 INFO HandlerThread:4861 [handler.py:finish():866] shutting down handler +2024-05-09 14:08:56,980 INFO WriterThread:4861 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/run-vqw25nkx.wandb +2024-05-09 14:08:56,980 INFO SenderThread:4861 [sender.py:finish():1546] shutting down sender diff --git a/wandb/latest-run/logs/debug.log b/wandb/latest-run/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..20f16f66df3d908b0be35c3bfcb0701e7bc181c6 --- /dev/null +++ b/wandb/latest-run/logs/debug.log @@ -0,0 +1,27 @@ +2024-05-09 14:01:23,067 INFO MainThread:4587 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Configure stats pid to 4587 +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug.log +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug-internal.log +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():561] calling init triggers +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():611] starting backend +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():615] setting up manager +2024-05-09 14:01:23,073 INFO MainThread:4587 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-09 14:01:23,075 INFO MainThread:4587 [wandb_init.py:init():623] backend started and connected +2024-05-09 14:01:23,086 INFO MainThread:4587 [wandb_init.py:init():715] updated telemetry +2024-05-09 14:01:23,094 INFO MainThread:4587 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-09 14:01:23,099 INFO MainThread:4587 [wandb_init.py:init():799] starting run threads in backend +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_console_start():2335] atexit reg +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-09 14:01:30,271 INFO MainThread:4587 [wandb_init.py:init():842] run started, returning control to user process +2024-05-09 14:08:56,984 WARNING MsgRouterThr:4587 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/latest-run/run-vqw25nkx.wandb b/wandb/latest-run/run-vqw25nkx.wandb new file mode 100644 index 0000000000000000000000000000000000000000..15b05152a0b8003cfd0f404922ca728881b09a02 Binary files /dev/null and b/wandb/latest-run/run-vqw25nkx.wandb differ diff --git a/wandb/offline-run-20240503_170937-00gisuah/files/conda-environment.yaml b/wandb/offline-run-20240503_170937-00gisuah/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..ff1e17786272fcd014c68c1dad0fd8a8d4c49752 --- /dev/null +++ b/wandb/offline-run-20240503_170937-00gisuah/files/conda-environment.yaml @@ -0,0 +1,615 @@ +name: base +channels: + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.6.0=h6a678d5_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2023.12.12=h06a4308_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - certifi=2024.2.2=py311h06a4308_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h06a4308_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.1=h5eee18b_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - fontconfig=2.14.1=h4c34cd2_2 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.1=h6a678d5_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libavif=0.11.1=h5eee18b_0 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=11.2.0=h1234567_1 + - libiconv=1.16=h7f8727e_2 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=11.2.0=h1234567_1 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libuuid=1.41.5=h5eee18b_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.0=py311ha02d727_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.0.13=h7f8727e_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.7=h955ad1f_0 + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.12=h1ccaba5_0 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - xarray=2023.6.0=py311h06a4308_0 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=h5eee18b_0 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.17.3 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.34.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240503_170937-00gisuah/files/wandb-metadata.json b/wandb/offline-run-20240503_170937-00gisuah/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8a97f367250efc01e56c08018d940db8a5eec00d --- /dev/null +++ b/wandb/offline-run-20240503_170937-00gisuah/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.7", + "heartbeatAt": "2024-05-04T00:09:37.949121", + "startedAt": "2024-05-04T00:09:37.748956", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-11n01.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.7999999999999998, + "min": 1500.0, + "max": 2800.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.501392364501953 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5838279724121 + } +} diff --git a/wandb/offline-run-20240503_170937-00gisuah/files/wandb-summary.json b/wandb/offline-run-20240503_170937-00gisuah/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..4cab2d787f922fafd371e8f23b9d4df2f6d91744 --- /dev/null +++ b/wandb/offline-run-20240503_170937-00gisuah/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 183}} \ No newline at end of file diff --git a/wandb/offline-run-20240503_170937-00gisuah/logs/debug-internal.log b/wandb/offline-run-20240503_170937-00gisuah/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4ef282ad116bb22cd1c87d03159dcb1089998750 --- /dev/null +++ b/wandb/offline-run-20240503_170937-00gisuah/logs/debug-internal.log @@ -0,0 +1,229 @@ +2024-05-03 17:09:37,808 INFO StreamThr :14507 [internal.py:wandb_internal():86] W&B internal server running at pid: 14507, started at: 2024-05-03 17:09:37.806281 +2024-05-03 17:09:37,811 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status +2024-05-03 17:09:37,818 INFO WriterThread:14507 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240503_170937-00gisuah/run-00gisuah.wandb +2024-05-03 17:09:37,893 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: run_start +2024-05-03 17:09:37,917 DEBUG HandlerThread:14507 [system_info.py:__init__():26] System info init +2024-05-03 17:09:37,917 DEBUG HandlerThread:14507 [system_info.py:__init__():41] System info init done +2024-05-03 17:09:37,917 INFO HandlerThread:14507 [system_monitor.py:start():194] Starting system monitor +2024-05-03 17:09:37,917 INFO SystemMonitor:14507 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-03 17:09:37,917 INFO HandlerThread:14507 [system_monitor.py:probe():214] Collecting system info +2024-05-03 17:09:37,918 INFO SystemMonitor:14507 [interfaces.py:start():190] Started cpu monitoring +2024-05-03 17:09:37,919 INFO SystemMonitor:14507 [interfaces.py:start():190] Started disk monitoring +2024-05-03 17:09:37,920 INFO SystemMonitor:14507 [interfaces.py:start():190] Started gpu monitoring +2024-05-03 17:09:37,920 INFO SystemMonitor:14507 [interfaces.py:start():190] Started memory monitoring +2024-05-03 17:09:37,921 INFO SystemMonitor:14507 [interfaces.py:start():190] Started network monitoring +2024-05-03 17:09:37,949 DEBUG HandlerThread:14507 [system_info.py:probe():150] Probing system +2024-05-03 17:09:37,951 DEBUG HandlerThread:14507 [system_info.py:_probe_git():135] Probing git +2024-05-03 17:09:37,964 DEBUG HandlerThread:14507 [system_info.py:_probe_git():143] Probing git done +2024-05-03 17:09:37,964 DEBUG HandlerThread:14507 [system_info.py:probe():198] Probing system done +2024-05-03 17:09:37,964 DEBUG HandlerThread:14507 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.7', 'heartbeatAt': '2024-05-04T00:09:37.949121', 'startedAt': '2024-05-04T00:09:37.748956', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-11n01.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.7999999999999998, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.501392364501953}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5838279724121}} +2024-05-03 17:09:37,964 INFO HandlerThread:14507 [system_monitor.py:probe():224] Finished collecting system info +2024-05-03 17:09:37,964 INFO HandlerThread:14507 [system_monitor.py:probe():227] Publishing system info +2024-05-03 17:09:37,965 DEBUG HandlerThread:14507 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-03 17:09:45,397 DEBUG HandlerThread:14507 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-03 17:09:45,403 INFO HandlerThread:14507 [system_monitor.py:probe():229] Finished publishing system info +2024-05-03 17:09:45,414 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:09:45,414 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: keepalive +2024-05-03 17:09:45,415 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:09:45,645 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: python_packages +2024-05-03 17:09:48,416 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:09:48,416 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:09:53,417 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:09:53,417 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:09:53,469 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:09:55,720 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:09:58,077 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:09:58,418 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:09:58,418 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:00,577 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:02,822 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:03,419 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:03,419 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:05,073 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:07,312 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:08,420 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:08,421 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:09,904 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:12,141 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:13,422 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:13,422 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:14,388 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:16,830 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:18,423 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:18,423 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:19,178 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:21,424 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:23,424 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:23,425 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:23,685 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:26,229 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:28,425 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:28,426 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:28,480 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:30,725 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:32,975 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:33,427 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:33,427 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:35,400 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:37,756 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:37,922 DEBUG SystemMonitor:14507 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-03 17:10:38,428 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:38,428 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:40,015 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:42,264 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:43,429 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:43,429 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:44,787 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:47,029 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:48,431 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:48,431 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:49,282 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:51,534 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:53,432 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:53,432 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:54,083 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:56,342 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:10:58,433 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:10:58,434 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:10:58,608 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: partial_history +2024-05-03 17:11:03,435 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:03,435 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:08,436 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:08,437 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:13,438 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:13,438 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:18,440 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:18,440 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:23,441 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:23,441 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:28,443 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:28,443 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:33,444 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:33,444 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:38,445 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:38,445 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:43,446 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:43,447 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:48,447 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:48,448 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:53,449 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:53,450 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:11:58,450 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:11:58,451 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:12:03,451 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:12:03,452 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:12:08,452 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:12:08,453 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:12:13,453 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:12:13,454 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:12:18,455 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:12:18,455 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:12:23,456 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:12:23,456 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:12:28,458 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:12:28,458 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:12:33,459 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:12:33,459 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:12:38,460 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:12:38,461 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:12:41,306 DEBUG SenderThread:14507 [sender.py:send():379] send: exit +2024-05-03 17:12:41,306 INFO SenderThread:14507 [sender.py:send_exit():586] handling exit code: 0 +2024-05-03 17:12:41,306 INFO SenderThread:14507 [sender.py:send_exit():588] handling runtime: 183 +2024-05-03 17:12:41,316 INFO SenderThread:14507 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-03 17:12:41,316 INFO SenderThread:14507 [sender.py:send_exit():594] send defer +2024-05-03 17:12:41,316 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,317 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 0 +2024-05-03 17:12:41,317 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,317 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-05-03 17:12:41,317 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 1 +2024-05-03 17:12:41,317 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,317 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 1 +2024-05-03 17:12:41,318 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,318 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-05-03 17:12:41,318 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 2 +2024-05-03 17:12:41,318 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,318 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 2 +2024-05-03 17:12:41,318 INFO HandlerThread:14507 [system_monitor.py:finish():203] Stopping system monitor +2024-05-03 17:12:41,318 DEBUG SystemMonitor:14507 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-03 17:12:41,319 DEBUG SystemMonitor:14507 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-03 17:12:41,321 INFO HandlerThread:14507 [interfaces.py:finish():202] Joined cpu monitor +2024-05-03 17:12:41,321 INFO HandlerThread:14507 [interfaces.py:finish():202] Joined disk monitor +2024-05-03 17:12:41,345 INFO HandlerThread:14507 [interfaces.py:finish():202] Joined gpu monitor +2024-05-03 17:12:41,346 INFO HandlerThread:14507 [interfaces.py:finish():202] Joined memory monitor +2024-05-03 17:12:41,346 INFO HandlerThread:14507 [interfaces.py:finish():202] Joined network monitor +2024-05-03 17:12:41,346 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,346 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-05-03 17:12:41,346 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 3 +2024-05-03 17:12:41,347 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,347 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 3 +2024-05-03 17:12:41,347 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,347 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-05-03 17:12:41,347 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 4 +2024-05-03 17:12:41,347 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,347 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 4 +2024-05-03 17:12:41,348 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,348 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-05-03 17:12:41,348 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 5 +2024-05-03 17:12:41,348 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,348 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 5 +2024-05-03 17:12:41,348 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,348 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-05-03 17:12:41,348 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 6 +2024-05-03 17:12:41,349 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,349 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 6 +2024-05-03 17:12:41,349 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,349 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-05-03 17:12:41,349 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 7 +2024-05-03 17:12:41,349 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: status_report +2024-05-03 17:12:41,349 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,349 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: status_report +2024-05-03 17:12:41,349 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 7 +2024-05-03 17:12:41,350 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,350 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-05-03 17:12:41,350 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 8 +2024-05-03 17:12:41,350 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,350 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 8 +2024-05-03 17:12:41,350 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,350 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-05-03 17:12:41,350 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 9 +2024-05-03 17:12:41,350 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,350 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 9 +2024-05-03 17:12:41,351 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,351 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-05-03 17:12:41,351 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 10 +2024-05-03 17:12:41,351 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,351 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 10 +2024-05-03 17:12:41,351 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,351 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-05-03 17:12:41,351 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 11 +2024-05-03 17:12:41,351 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,351 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 11 +2024-05-03 17:12:41,351 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,351 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-05-03 17:12:41,352 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 12 +2024-05-03 17:12:41,352 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,352 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 12 +2024-05-03 17:12:41,352 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,352 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-05-03 17:12:41,352 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 13 +2024-05-03 17:12:41,352 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,352 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 13 +2024-05-03 17:12:41,352 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,352 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-05-03 17:12:41,352 INFO SenderThread:14507 [sender.py:transition_state():614] send defer: 14 +2024-05-03 17:12:41,353 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: defer +2024-05-03 17:12:41,353 DEBUG SenderThread:14507 [sender.py:send():379] send: final +2024-05-03 17:12:41,353 INFO HandlerThread:14507 [handler.py:handle_request_defer():172] handle defer: 14 +2024-05-03 17:12:41,353 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: defer +2024-05-03 17:12:41,353 INFO SenderThread:14507 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-05-03 17:12:41,354 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-03 17:12:41,354 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: poll_exit +2024-05-03 17:12:41,355 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-03 17:12:41,356 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: server_info +2024-05-03 17:12:41,356 DEBUG SenderThread:14507 [sender.py:send_request():406] send_request: server_info +2024-05-03 17:12:41,357 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: get_summary +2024-05-03 17:12:41,357 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: sampled_history +2024-05-03 17:12:41,397 DEBUG HandlerThread:14507 [handler.py:handle_request():146] handle_request: shutdown +2024-05-03 17:12:41,397 INFO HandlerThread:14507 [handler.py:finish():866] shutting down handler +2024-05-03 17:12:42,356 INFO WriterThread:14507 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240503_170937-00gisuah/run-00gisuah.wandb +2024-05-03 17:12:42,356 INFO SenderThread:14507 [sender.py:finish():1546] shutting down sender diff --git a/wandb/offline-run-20240503_170937-00gisuah/logs/debug.log b/wandb/offline-run-20240503_170937-00gisuah/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..5b0f64df8b168b3d93d0ed4d2d80bd71be4a8130 --- /dev/null +++ b/wandb/offline-run-20240503_170937-00gisuah/logs/debug.log @@ -0,0 +1,32 @@ +2024-05-03 17:09:37,782 INFO MainThread:13922 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-03 17:09:37,784 INFO MainThread:13922 [wandb_setup.py:_flush():76] Configure stats pid to 13922 +2024-05-03 17:09:37,784 INFO MainThread:13922 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-03 17:09:37,784 INFO MainThread:13922 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-03 17:09:37,784 INFO MainThread:13922 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-03 17:09:37,784 INFO MainThread:13922 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-03 17:09:37,784 INFO MainThread:13922 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-03 17:09:37,784 INFO MainThread:13922 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-03 17:09:37,784 INFO MainThread:13922 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-03 17:09:37,784 INFO MainThread:13922 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240503_170937-00gisuah/logs/debug.log +2024-05-03 17:09:37,785 INFO MainThread:13922 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240503_170937-00gisuah/logs/debug-internal.log +2024-05-03 17:09:37,785 INFO MainThread:13922 [wandb_init.py:init():561] calling init triggers +2024-05-03 17:09:37,785 INFO MainThread:13922 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-03 17:09:37,785 INFO MainThread:13922 [wandb_init.py:init():611] starting backend +2024-05-03 17:09:37,785 INFO MainThread:13922 [wandb_init.py:init():615] setting up manager +2024-05-03 17:09:37,804 INFO MainThread:13922 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-03 17:09:37,809 INFO MainThread:13922 [wandb_init.py:init():623] backend started and connected +2024-05-03 17:09:37,818 INFO MainThread:13922 [wandb_init.py:init():715] updated telemetry +2024-05-03 17:09:37,841 INFO MainThread:13922 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-03 17:09:37,848 INFO MainThread:13922 [wandb_init.py:init():799] starting run threads in backend +2024-05-03 17:09:45,644 INFO MainThread:13922 [wandb_run.py:_console_start():2335] atexit reg +2024-05-03 17:09:45,644 INFO MainThread:13922 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-03 17:09:45,644 INFO MainThread:13922 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-03 17:09:45,645 INFO MainThread:13922 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-03 17:09:45,647 INFO MainThread:13922 [wandb_init.py:init():842] run started, returning control to user process +2024-05-03 17:12:41,300 INFO MainThread:13922 [wandb_run.py:_finish():2064] finishing run stanford-voltron/onyx-vlms/00gisuah +2024-05-03 17:12:41,301 INFO MainThread:13922 [wandb_run.py:_atexit_cleanup():2304] got exitcode: 0 +2024-05-03 17:12:41,305 INFO MainThread:13922 [wandb_run.py:_restore():2287] restore +2024-05-03 17:12:41,305 INFO MainThread:13922 [wandb_run.py:_restore():2293] restore done +2024-05-03 17:12:42,399 INFO MainThread:13922 [wandb_run.py:_footer_history_summary_info():3936] rendering history +2024-05-03 17:12:42,400 INFO MainThread:13922 [wandb_run.py:_footer_history_summary_info():3968] rendering summary diff --git a/wandb/offline-run-20240503_170937-00gisuah/run-00gisuah.wandb b/wandb/offline-run-20240503_170937-00gisuah/run-00gisuah.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ef09191fa1e63d0c81638939d3e20d4d9a0711d4 Binary files /dev/null and b/wandb/offline-run-20240503_170937-00gisuah/run-00gisuah.wandb differ diff --git a/wandb/offline-run-20240506_123154-8qgmomje/files/conda-environment.yaml b/wandb/offline-run-20240506_123154-8qgmomje/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39001b8b031536cfd545c38c20a0fa75f5542dc2 --- /dev/null +++ b/wandb/offline-run-20240506_123154-8qgmomje/files/conda-environment.yaml @@ -0,0 +1,615 @@ +name: base +channels: + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.6.0=h6a678d5_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2023.12.12=h06a4308_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - certifi=2024.2.2=py311h06a4308_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h06a4308_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.1=h5eee18b_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - fontconfig=2.14.1=h4c34cd2_2 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.1=h6a678d5_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libavif=0.11.1=h5eee18b_0 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=11.2.0=h1234567_1 + - libiconv=1.16=h7f8727e_2 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=11.2.0=h1234567_1 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libuuid=1.41.5=h5eee18b_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.0=py311ha02d727_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.0.13=h7f8727e_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.7=h955ad1f_0 + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.12=h1ccaba5_0 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - xarray=2023.6.0=py311h06a4308_0 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=h5eee18b_0 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.34.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240506_123154-8qgmomje/files/wandb-metadata.json b/wandb/offline-run-20240506_123154-8qgmomje/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a48a7a5696c7f641ebd1c6effae92e4ad5f051e7 --- /dev/null +++ b/wandb/offline-run-20240506_123154-8qgmomje/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.7", + "heartbeatAt": "2024-05-06T19:31:54.807774", + "startedAt": "2024-05-06T19:31:54.630001", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 2.25625, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 11.855579376220703 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240506_123154-8qgmomje/files/wandb-summary.json b/wandb/offline-run-20240506_123154-8qgmomje/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..19c891fd85f4b70019c5601c810946791e4c2b75 --- /dev/null +++ b/wandb/offline-run-20240506_123154-8qgmomje/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 196}} \ No newline at end of file diff --git a/wandb/offline-run-20240506_123154-8qgmomje/logs/debug-internal.log b/wandb/offline-run-20240506_123154-8qgmomje/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..053fecce8212471ba26d5dfef98018be1b4b0c81 --- /dev/null +++ b/wandb/offline-run-20240506_123154-8qgmomje/logs/debug-internal.log @@ -0,0 +1,235 @@ +2024-05-06 12:31:54,656 INFO StreamThr :33656 [internal.py:wandb_internal():86] W&B internal server running at pid: 33656, started at: 2024-05-06 12:31:54.653817 +2024-05-06 12:31:54,658 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status +2024-05-06 12:31:54,673 INFO WriterThread:33656 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240506_123154-8qgmomje/run-8qgmomje.wandb +2024-05-06 12:31:54,751 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: run_start +2024-05-06 12:31:54,775 DEBUG HandlerThread:33656 [system_info.py:__init__():26] System info init +2024-05-06 12:31:54,775 DEBUG HandlerThread:33656 [system_info.py:__init__():41] System info init done +2024-05-06 12:31:54,775 INFO HandlerThread:33656 [system_monitor.py:start():194] Starting system monitor +2024-05-06 12:31:54,776 INFO SystemMonitor:33656 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-06 12:31:54,776 INFO HandlerThread:33656 [system_monitor.py:probe():214] Collecting system info +2024-05-06 12:31:54,777 INFO SystemMonitor:33656 [interfaces.py:start():190] Started cpu monitoring +2024-05-06 12:31:54,777 INFO SystemMonitor:33656 [interfaces.py:start():190] Started disk monitoring +2024-05-06 12:31:54,778 INFO SystemMonitor:33656 [interfaces.py:start():190] Started gpu monitoring +2024-05-06 12:31:54,779 INFO SystemMonitor:33656 [interfaces.py:start():190] Started memory monitoring +2024-05-06 12:31:54,781 INFO SystemMonitor:33656 [interfaces.py:start():190] Started network monitoring +2024-05-06 12:31:54,807 DEBUG HandlerThread:33656 [system_info.py:probe():150] Probing system +2024-05-06 12:31:54,812 DEBUG HandlerThread:33656 [system_info.py:_probe_git():135] Probing git +2024-05-06 12:31:54,825 DEBUG HandlerThread:33656 [system_info.py:_probe_git():143] Probing git done +2024-05-06 12:31:54,825 DEBUG HandlerThread:33656 [system_info.py:probe():198] Probing system done +2024-05-06 12:31:54,825 DEBUG HandlerThread:33656 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.7', 'heartbeatAt': '2024-05-06T19:31:54.807774', 'startedAt': '2024-05-06T19:31:54.630001', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 2.25625, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 11.855579376220703}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-06 12:31:54,826 INFO HandlerThread:33656 [system_monitor.py:probe():224] Finished collecting system info +2024-05-06 12:31:54,826 INFO HandlerThread:33656 [system_monitor.py:probe():227] Publishing system info +2024-05-06 12:31:54,826 DEBUG HandlerThread:33656 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-06 12:32:04,210 DEBUG HandlerThread:33656 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-06 12:32:04,214 INFO HandlerThread:33656 [system_monitor.py:probe():229] Finished publishing system info +2024-05-06 12:32:04,229 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:04,229 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: keepalive +2024-05-06 12:32:04,230 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:05,231 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:05,231 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:05,476 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: python_packages +2024-05-06 12:32:10,232 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:10,232 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:15,047 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:15,233 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:15,234 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:17,526 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:20,128 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:20,235 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:20,235 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:22,887 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:25,236 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:25,237 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:25,332 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:27,800 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:30,238 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:30,239 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:30,258 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:33,073 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:35,240 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:35,240 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:35,523 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:37,987 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:40,241 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:40,242 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:40,569 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:43,266 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:45,243 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:45,243 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:45,705 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:48,153 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:50,244 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:50,244 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:50,718 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:53,399 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:54,781 DEBUG SystemMonitor:33656 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-06 12:32:55,245 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:32:55,246 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:32:55,838 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:32:58,268 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:00,247 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:00,247 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:00,724 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:03,534 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:05,248 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:05,248 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:05,985 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:08,433 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:10,249 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:10,250 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:10,992 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:13,663 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:15,251 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:15,251 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:16,118 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:18,584 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:20,252 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:20,252 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:21,023 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:23,827 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:25,253 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:25,253 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:26,268 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: partial_history +2024-05-06 12:33:30,254 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:30,255 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:35,256 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:35,256 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:40,257 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:40,257 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:45,258 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:45,258 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:50,259 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:50,260 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:33:55,260 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:33:55,261 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:00,262 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:00,262 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:05,263 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:05,264 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:10,264 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:10,265 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:15,266 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:15,266 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:20,267 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:20,267 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:25,268 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:25,269 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:30,270 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:30,270 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:35,271 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:35,271 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:40,272 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:40,272 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:45,273 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:45,274 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:50,275 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:50,275 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:34:55,276 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:34:55,276 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:35:00,277 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:35:00,277 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:35:05,278 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:35:05,279 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:35:10,280 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:35:10,280 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:35:11,207 DEBUG SenderThread:33656 [sender.py:send():379] send: exit +2024-05-06 12:35:11,208 INFO SenderThread:33656 [sender.py:send_exit():586] handling exit code: 0 +2024-05-06 12:35:11,208 INFO SenderThread:33656 [sender.py:send_exit():588] handling runtime: 196 +2024-05-06 12:35:11,211 INFO SenderThread:33656 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-06 12:35:11,211 INFO SenderThread:33656 [sender.py:send_exit():594] send defer +2024-05-06 12:35:11,211 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,211 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 0 +2024-05-06 12:35:11,211 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,212 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-05-06 12:35:11,212 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 1 +2024-05-06 12:35:11,212 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,212 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 1 +2024-05-06 12:35:11,212 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,212 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-05-06 12:35:11,212 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 2 +2024-05-06 12:35:11,212 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,213 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 2 +2024-05-06 12:35:11,213 INFO HandlerThread:33656 [system_monitor.py:finish():203] Stopping system monitor +2024-05-06 12:35:11,213 DEBUG SystemMonitor:33656 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-06 12:35:11,213 DEBUG SystemMonitor:33656 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-06 12:35:11,214 INFO HandlerThread:33656 [interfaces.py:finish():202] Joined cpu monitor +2024-05-06 12:35:11,216 INFO HandlerThread:33656 [interfaces.py:finish():202] Joined disk monitor +2024-05-06 12:35:11,256 INFO HandlerThread:33656 [interfaces.py:finish():202] Joined gpu monitor +2024-05-06 12:35:11,256 INFO HandlerThread:33656 [interfaces.py:finish():202] Joined memory monitor +2024-05-06 12:35:11,256 INFO HandlerThread:33656 [interfaces.py:finish():202] Joined network monitor +2024-05-06 12:35:11,257 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,257 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-05-06 12:35:11,257 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 3 +2024-05-06 12:35:11,257 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,257 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 3 +2024-05-06 12:35:11,258 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,258 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-05-06 12:35:11,258 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 4 +2024-05-06 12:35:11,259 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,259 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 4 +2024-05-06 12:35:11,259 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,259 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-05-06 12:35:11,259 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 5 +2024-05-06 12:35:11,259 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,259 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 5 +2024-05-06 12:35:11,260 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,260 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-05-06 12:35:11,260 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 6 +2024-05-06 12:35:11,260 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,260 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 6 +2024-05-06 12:35:11,260 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,261 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-05-06 12:35:11,261 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 7 +2024-05-06 12:35:11,261 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 12:35:11,261 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,261 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 7 +2024-05-06 12:35:11,261 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: status_report +2024-05-06 12:35:11,261 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,262 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-05-06 12:35:11,262 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 8 +2024-05-06 12:35:11,262 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,262 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 8 +2024-05-06 12:35:11,262 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,262 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-05-06 12:35:11,262 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 9 +2024-05-06 12:35:11,262 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,262 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 9 +2024-05-06 12:35:11,263 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,263 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-05-06 12:35:11,263 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 10 +2024-05-06 12:35:11,263 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,263 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 10 +2024-05-06 12:35:11,263 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,263 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-05-06 12:35:11,263 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 11 +2024-05-06 12:35:11,264 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,264 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 11 +2024-05-06 12:35:11,264 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,264 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-05-06 12:35:11,264 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 12 +2024-05-06 12:35:11,264 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,264 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 12 +2024-05-06 12:35:11,264 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,265 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-05-06 12:35:11,265 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 13 +2024-05-06 12:35:11,265 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,265 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 13 +2024-05-06 12:35:11,265 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,265 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-05-06 12:35:11,265 INFO SenderThread:33656 [sender.py:transition_state():614] send defer: 14 +2024-05-06 12:35:11,266 DEBUG SenderThread:33656 [sender.py:send():379] send: final +2024-05-06 12:35:11,266 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: defer +2024-05-06 12:35:11,266 INFO HandlerThread:33656 [handler.py:handle_request_defer():172] handle defer: 14 +2024-05-06 12:35:11,266 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: defer +2024-05-06 12:35:11,266 INFO SenderThread:33656 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-05-06 12:35:11,268 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-06 12:35:11,268 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: poll_exit +2024-05-06 12:35:11,269 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-06 12:35:11,270 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: server_info +2024-05-06 12:35:11,270 DEBUG SenderThread:33656 [sender.py:send_request():406] send_request: server_info +2024-05-06 12:35:11,271 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: get_summary +2024-05-06 12:35:11,271 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: sampled_history +2024-05-06 12:35:11,312 DEBUG HandlerThread:33656 [handler.py:handle_request():146] handle_request: shutdown +2024-05-06 12:35:11,312 INFO HandlerThread:33656 [handler.py:finish():866] shutting down handler +2024-05-06 12:35:12,270 INFO WriterThread:33656 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240506_123154-8qgmomje/run-8qgmomje.wandb +2024-05-06 12:35:12,270 INFO SenderThread:33656 [sender.py:finish():1546] shutting down sender diff --git a/wandb/offline-run-20240506_123154-8qgmomje/logs/debug.log b/wandb/offline-run-20240506_123154-8qgmomje/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..e2efc407ae0993fc69e35da45cab666876e54156 --- /dev/null +++ b/wandb/offline-run-20240506_123154-8qgmomje/logs/debug.log @@ -0,0 +1,32 @@ +2024-05-06 12:31:54,640 INFO MainThread:32269 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-06 12:31:54,641 INFO MainThread:32269 [wandb_setup.py:_flush():76] Configure stats pid to 32269 +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240506_123154-8qgmomje/logs/debug.log +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240506_123154-8qgmomje/logs/debug-internal.log +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_init.py:init():561] calling init triggers +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-06 12:31:54,642 INFO MainThread:32269 [wandb_init.py:init():611] starting backend +2024-05-06 12:31:54,643 INFO MainThread:32269 [wandb_init.py:init():615] setting up manager +2024-05-06 12:31:54,651 INFO MainThread:32269 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-06 12:31:54,661 INFO MainThread:32269 [wandb_init.py:init():623] backend started and connected +2024-05-06 12:31:54,672 INFO MainThread:32269 [wandb_init.py:init():715] updated telemetry +2024-05-06 12:31:54,699 INFO MainThread:32269 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-06 12:31:54,706 INFO MainThread:32269 [wandb_init.py:init():799] starting run threads in backend +2024-05-06 12:32:05,476 INFO MainThread:32269 [wandb_run.py:_console_start():2335] atexit reg +2024-05-06 12:32:05,476 INFO MainThread:32269 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-06 12:32:05,476 INFO MainThread:32269 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-06 12:32:05,477 INFO MainThread:32269 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-06 12:32:05,478 INFO MainThread:32269 [wandb_init.py:init():842] run started, returning control to user process +2024-05-06 12:35:11,205 INFO MainThread:32269 [wandb_run.py:_finish():2064] finishing run stanford-voltron/onyx-vlms/8qgmomje +2024-05-06 12:35:11,206 INFO MainThread:32269 [wandb_run.py:_atexit_cleanup():2304] got exitcode: 0 +2024-05-06 12:35:11,207 INFO MainThread:32269 [wandb_run.py:_restore():2287] restore +2024-05-06 12:35:11,207 INFO MainThread:32269 [wandb_run.py:_restore():2293] restore done +2024-05-06 12:35:12,314 INFO MainThread:32269 [wandb_run.py:_footer_history_summary_info():3936] rendering history +2024-05-06 12:35:12,315 INFO MainThread:32269 [wandb_run.py:_footer_history_summary_info():3968] rendering summary diff --git a/wandb/offline-run-20240506_123154-8qgmomje/run-8qgmomje.wandb b/wandb/offline-run-20240506_123154-8qgmomje/run-8qgmomje.wandb new file mode 100644 index 0000000000000000000000000000000000000000..976acdce7d4ef88613ad6e6d3e35461f953036cf Binary files /dev/null and b/wandb/offline-run-20240506_123154-8qgmomje/run-8qgmomje.wandb differ diff --git a/wandb/offline-run-20240506_160400-io481435/files/conda-environment.yaml b/wandb/offline-run-20240506_160400-io481435/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..4562ef3c316b8482e3842c8827cf2433ea513981 --- /dev/null +++ b/wandb/offline-run-20240506_160400-io481435/files/conda-environment.yaml @@ -0,0 +1,615 @@ +name: base +channels: + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.6.0=h6a678d5_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2023.12.12=h06a4308_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - certifi=2024.2.2=py311h06a4308_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h06a4308_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.1=h5eee18b_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - fontconfig=2.14.1=h4c34cd2_2 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.1=h6a678d5_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libavif=0.11.1=h5eee18b_0 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=11.2.0=h1234567_1 + - libiconv=1.16=h7f8727e_2 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=11.2.0=h1234567_1 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libuuid=1.41.5=h5eee18b_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.0=py311ha02d727_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.0.13=h7f8727e_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.7=h955ad1f_0 + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.12=h1ccaba5_0 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - xarray=2023.6.0=py311h06a4308_0 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=h5eee18b_0 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.31.0 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240506_160400-io481435/files/wandb-metadata.json b/wandb/offline-run-20240506_160400-io481435/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..d7fb1735eaf69d4232d66d01f31858305052b299 --- /dev/null +++ b/wandb/offline-run-20240506_160400-io481435/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.7", + "heartbeatAt": "2024-05-06T23:04:00.890688", + "startedAt": "2024-05-06T23:04:00.701540", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 2.2609375, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 11.856884002685547 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240506_160400-io481435/logs/debug-internal.log b/wandb/offline-run-20240506_160400-io481435/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..6966e40224db316fa2dd9434fc45e04919b8d15d --- /dev/null +++ b/wandb/offline-run-20240506_160400-io481435/logs/debug-internal.log @@ -0,0 +1,31 @@ +2024-05-06 16:04:00,754 INFO StreamThr :27791 [internal.py:wandb_internal():86] W&B internal server running at pid: 27791, started at: 2024-05-06 16:04:00.748270 +2024-05-06 16:04:00,760 DEBUG HandlerThread:27791 [handler.py:handle_request():146] handle_request: status +2024-05-06 16:04:00,761 INFO WriterThread:27791 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240506_160400-io481435/run-io481435.wandb +2024-05-06 16:04:00,830 DEBUG HandlerThread:27791 [handler.py:handle_request():146] handle_request: run_start +2024-05-06 16:04:00,856 DEBUG HandlerThread:27791 [system_info.py:__init__():26] System info init +2024-05-06 16:04:00,857 DEBUG HandlerThread:27791 [system_info.py:__init__():41] System info init done +2024-05-06 16:04:00,857 INFO HandlerThread:27791 [system_monitor.py:start():194] Starting system monitor +2024-05-06 16:04:00,857 INFO SystemMonitor:27791 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-06 16:04:00,857 INFO HandlerThread:27791 [system_monitor.py:probe():214] Collecting system info +2024-05-06 16:04:00,858 INFO SystemMonitor:27791 [interfaces.py:start():190] Started cpu monitoring +2024-05-06 16:04:00,860 INFO SystemMonitor:27791 [interfaces.py:start():190] Started disk monitoring +2024-05-06 16:04:00,860 INFO SystemMonitor:27791 [interfaces.py:start():190] Started gpu monitoring +2024-05-06 16:04:00,861 INFO SystemMonitor:27791 [interfaces.py:start():190] Started memory monitoring +2024-05-06 16:04:00,862 INFO SystemMonitor:27791 [interfaces.py:start():190] Started network monitoring +2024-05-06 16:04:00,890 DEBUG HandlerThread:27791 [system_info.py:probe():150] Probing system +2024-05-06 16:04:00,894 DEBUG HandlerThread:27791 [system_info.py:_probe_git():135] Probing git +2024-05-06 16:04:00,909 DEBUG HandlerThread:27791 [system_info.py:_probe_git():143] Probing git done +2024-05-06 16:04:00,910 DEBUG HandlerThread:27791 [system_info.py:probe():198] Probing system done +2024-05-06 16:04:00,910 DEBUG HandlerThread:27791 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.7', 'heartbeatAt': '2024-05-06T23:04:00.890688', 'startedAt': '2024-05-06T23:04:00.701540', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 2.2609375, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 11.856884002685547}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-06 16:04:00,910 INFO HandlerThread:27791 [system_monitor.py:probe():224] Finished collecting system info +2024-05-06 16:04:00,910 INFO HandlerThread:27791 [system_monitor.py:probe():227] Publishing system info +2024-05-06 16:04:00,911 DEBUG HandlerThread:27791 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-06 16:04:10,655 DEBUG HandlerThread:27791 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-06 16:04:10,662 INFO HandlerThread:27791 [system_monitor.py:probe():229] Finished publishing system info +2024-05-06 16:04:10,674 DEBUG HandlerThread:27791 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 16:04:10,674 DEBUG HandlerThread:27791 [handler.py:handle_request():146] handle_request: keepalive +2024-05-06 16:04:10,675 DEBUG SenderThread:27791 [sender.py:send_request():406] send_request: status_report +2024-05-06 16:04:11,676 DEBUG HandlerThread:27791 [handler.py:handle_request():146] handle_request: status_report +2024-05-06 16:04:11,677 DEBUG SenderThread:27791 [sender.py:send_request():406] send_request: status_report +2024-05-06 16:04:12,003 DEBUG HandlerThread:27791 [handler.py:handle_request():146] handle_request: python_packages +2024-05-06 16:04:14,797 INFO MainThread:27791 [internal.py:handle_exit():76] Internal process exited diff --git a/wandb/offline-run-20240506_160400-io481435/logs/debug.log b/wandb/offline-run-20240506_160400-io481435/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..08f3b99d72ca8fae9fdca5106e331c0cf64969c6 --- /dev/null +++ b/wandb/offline-run-20240506_160400-io481435/logs/debug.log @@ -0,0 +1,26 @@ +2024-05-06 16:04:00,714 INFO MainThread:27535 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-06 16:04:00,715 INFO MainThread:27535 [wandb_setup.py:_flush():76] Configure stats pid to 27535 +2024-05-06 16:04:00,715 INFO MainThread:27535 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240506_160400-io481435/logs/debug.log +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240506_160400-io481435/logs/debug-internal.log +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_init.py:init():561] calling init triggers +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_init.py:init():611] starting backend +2024-05-06 16:04:00,716 INFO MainThread:27535 [wandb_init.py:init():615] setting up manager +2024-05-06 16:04:00,721 INFO MainThread:27535 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-06 16:04:00,739 INFO MainThread:27535 [wandb_init.py:init():623] backend started and connected +2024-05-06 16:04:00,746 INFO MainThread:27535 [wandb_init.py:init():715] updated telemetry +2024-05-06 16:04:00,776 INFO MainThread:27535 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-06 16:04:00,785 INFO MainThread:27535 [wandb_init.py:init():799] starting run threads in backend +2024-05-06 16:04:12,002 INFO MainThread:27535 [wandb_run.py:_console_start():2335] atexit reg +2024-05-06 16:04:12,002 INFO MainThread:27535 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-06 16:04:12,003 INFO MainThread:27535 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-06 16:04:12,003 INFO MainThread:27535 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-06 16:04:12,004 INFO MainThread:27535 [wandb_init.py:init():842] run started, returning control to user process diff --git a/wandb/offline-run-20240506_160400-io481435/run-io481435.wandb b/wandb/offline-run-20240506_160400-io481435/run-io481435.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/wandb/offline-run-20240507_152956-gy8tfquj/files/conda-environment.yaml b/wandb/offline-run-20240507_152956-gy8tfquj/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..39001b8b031536cfd545c38c20a0fa75f5542dc2 --- /dev/null +++ b/wandb/offline-run-20240507_152956-gy8tfquj/files/conda-environment.yaml @@ -0,0 +1,615 @@ +name: base +channels: + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.6.0=h6a678d5_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2023.12.12=h06a4308_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - certifi=2024.2.2=py311h06a4308_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h06a4308_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.1=h5eee18b_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - fontconfig=2.14.1=h4c34cd2_2 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.1=h6a678d5_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libavif=0.11.1=h5eee18b_0 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=11.2.0=h1234567_1 + - libiconv=1.16=h7f8727e_2 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=11.2.0=h1234567_1 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libuuid=1.41.5=h5eee18b_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.0=py311ha02d727_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.0.13=h7f8727e_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.7=h955ad1f_0 + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.12=h1ccaba5_0 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - xarray=2023.6.0=py311h06a4308_0 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=h5eee18b_0 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.34.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240507_152956-gy8tfquj/files/wandb-metadata.json b/wandb/offline-run-20240507_152956-gy8tfquj/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..6e28de96182c5f42f4721dcae2f3fe8c117eab7d --- /dev/null +++ b/wandb/offline-run-20240507_152956-gy8tfquj/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.7", + "heartbeatAt": "2024-05-07T22:29:56.551891", + "startedAt": "2024-05-07T22:29:56.381331", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.584375, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.2, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.2191314697265625 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240507_152956-gy8tfquj/files/wandb-summary.json b/wandb/offline-run-20240507_152956-gy8tfquj/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..b1ac4f7d3564b2fd407d247e6957709faa41a169 --- /dev/null +++ b/wandb/offline-run-20240507_152956-gy8tfquj/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 9}} \ No newline at end of file diff --git a/wandb/offline-run-20240507_152956-gy8tfquj/logs/debug-internal.log b/wandb/offline-run-20240507_152956-gy8tfquj/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..a7e112c9e0475ef30f2e09fd3e44b2308d7cd30f --- /dev/null +++ b/wandb/offline-run-20240507_152956-gy8tfquj/logs/debug-internal.log @@ -0,0 +1,133 @@ +2024-05-07 15:29:56,404 INFO StreamThr :7128 [internal.py:wandb_internal():86] W&B internal server running at pid: 7128, started at: 2024-05-07 15:29:56.401592 +2024-05-07 15:29:56,407 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: status +2024-05-07 15:29:56,416 INFO WriterThread:7128 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240507_152956-gy8tfquj/run-gy8tfquj.wandb +2024-05-07 15:29:56,491 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: run_start +2024-05-07 15:29:56,514 DEBUG HandlerThread:7128 [system_info.py:__init__():26] System info init +2024-05-07 15:29:56,514 DEBUG HandlerThread:7128 [system_info.py:__init__():41] System info init done +2024-05-07 15:29:56,514 INFO HandlerThread:7128 [system_monitor.py:start():194] Starting system monitor +2024-05-07 15:29:56,515 INFO SystemMonitor:7128 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-07 15:29:56,515 INFO HandlerThread:7128 [system_monitor.py:probe():214] Collecting system info +2024-05-07 15:29:56,516 INFO SystemMonitor:7128 [interfaces.py:start():190] Started cpu monitoring +2024-05-07 15:29:56,517 INFO SystemMonitor:7128 [interfaces.py:start():190] Started disk monitoring +2024-05-07 15:29:56,519 INFO SystemMonitor:7128 [interfaces.py:start():190] Started gpu monitoring +2024-05-07 15:29:56,520 INFO SystemMonitor:7128 [interfaces.py:start():190] Started memory monitoring +2024-05-07 15:29:56,521 INFO SystemMonitor:7128 [interfaces.py:start():190] Started network monitoring +2024-05-07 15:29:56,551 DEBUG HandlerThread:7128 [system_info.py:probe():150] Probing system +2024-05-07 15:29:56,555 DEBUG HandlerThread:7128 [system_info.py:_probe_git():135] Probing git +2024-05-07 15:29:56,568 DEBUG HandlerThread:7128 [system_info.py:_probe_git():143] Probing git done +2024-05-07 15:29:56,568 DEBUG HandlerThread:7128 [system_info.py:probe():198] Probing system done +2024-05-07 15:29:56,568 DEBUG HandlerThread:7128 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.7', 'heartbeatAt': '2024-05-07T22:29:56.551891', 'startedAt': '2024-05-07T22:29:56.381331', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.584375, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.2, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.2191314697265625}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-07 15:29:56,568 INFO HandlerThread:7128 [system_monitor.py:probe():224] Finished collecting system info +2024-05-07 15:29:56,568 INFO HandlerThread:7128 [system_monitor.py:probe():227] Publishing system info +2024-05-07 15:29:56,568 DEBUG HandlerThread:7128 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-07 15:30:05,475 DEBUG HandlerThread:7128 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-07 15:30:05,479 INFO HandlerThread:7128 [system_monitor.py:probe():229] Finished publishing system info +2024-05-07 15:30:05,491 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: status_report +2024-05-07 15:30:05,491 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: keepalive +2024-05-07 15:30:05,492 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: status_report +2024-05-07 15:30:05,922 DEBUG SenderThread:7128 [sender.py:send():379] send: exit +2024-05-07 15:30:05,923 INFO SenderThread:7128 [sender.py:send_exit():586] handling exit code: 255 +2024-05-07 15:30:05,923 INFO SenderThread:7128 [sender.py:send_exit():588] handling runtime: 9 +2024-05-07 15:30:05,926 INFO SenderThread:7128 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-07 15:30:05,926 INFO SenderThread:7128 [sender.py:send_exit():594] send defer +2024-05-07 15:30:05,926 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,926 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 0 +2024-05-07 15:30:05,926 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,927 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-05-07 15:30:05,927 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 1 +2024-05-07 15:30:05,927 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,927 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 1 +2024-05-07 15:30:05,927 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,927 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-05-07 15:30:05,927 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 2 +2024-05-07 15:30:05,927 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,927 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 2 +2024-05-07 15:30:05,928 INFO HandlerThread:7128 [system_monitor.py:finish():203] Stopping system monitor +2024-05-07 15:30:05,928 DEBUG SystemMonitor:7128 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-07 15:30:05,929 DEBUG SystemMonitor:7128 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-07 15:30:05,929 INFO HandlerThread:7128 [interfaces.py:finish():202] Joined cpu monitor +2024-05-07 15:30:05,929 DEBUG SystemMonitor:7128 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-07 15:30:05,929 INFO HandlerThread:7128 [interfaces.py:finish():202] Joined disk monitor +2024-05-07 15:30:05,961 INFO HandlerThread:7128 [interfaces.py:finish():202] Joined gpu monitor +2024-05-07 15:30:05,962 INFO HandlerThread:7128 [interfaces.py:finish():202] Joined memory monitor +2024-05-07 15:30:05,962 INFO HandlerThread:7128 [interfaces.py:finish():202] Joined network monitor +2024-05-07 15:30:05,963 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,963 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-05-07 15:30:05,963 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 3 +2024-05-07 15:30:05,963 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,964 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 3 +2024-05-07 15:30:05,964 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,964 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-05-07 15:30:05,964 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 4 +2024-05-07 15:30:05,964 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,964 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 4 +2024-05-07 15:30:05,964 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,964 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-05-07 15:30:05,964 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 5 +2024-05-07 15:30:05,965 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,965 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 5 +2024-05-07 15:30:05,965 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,965 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-05-07 15:30:05,965 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 6 +2024-05-07 15:30:05,965 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,965 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 6 +2024-05-07 15:30:05,966 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,966 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-05-07 15:30:05,966 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 7 +2024-05-07 15:30:05,966 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: status_report +2024-05-07 15:30:05,966 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,966 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 7 +2024-05-07 15:30:05,966 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: status_report +2024-05-07 15:30:05,967 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,967 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-05-07 15:30:05,967 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 8 +2024-05-07 15:30:05,967 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,967 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 8 +2024-05-07 15:30:05,967 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,967 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-05-07 15:30:05,967 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 9 +2024-05-07 15:30:05,967 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,968 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 9 +2024-05-07 15:30:05,968 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,968 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-05-07 15:30:05,968 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 10 +2024-05-07 15:30:05,968 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,968 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 10 +2024-05-07 15:30:05,968 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,968 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-05-07 15:30:05,968 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 11 +2024-05-07 15:30:05,969 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,969 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 11 +2024-05-07 15:30:05,969 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,969 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-05-07 15:30:05,969 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 12 +2024-05-07 15:30:05,969 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,969 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 12 +2024-05-07 15:30:05,969 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,969 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-05-07 15:30:05,970 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 13 +2024-05-07 15:30:05,970 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,970 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 13 +2024-05-07 15:30:05,970 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,970 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-05-07 15:30:05,970 INFO SenderThread:7128 [sender.py:transition_state():614] send defer: 14 +2024-05-07 15:30:05,971 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: defer +2024-05-07 15:30:05,971 DEBUG SenderThread:7128 [sender.py:send():379] send: final +2024-05-07 15:30:05,971 INFO HandlerThread:7128 [handler.py:handle_request_defer():172] handle defer: 14 +2024-05-07 15:30:05,971 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: defer +2024-05-07 15:30:05,971 INFO SenderThread:7128 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-05-07 15:30:05,972 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-07 15:30:05,972 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: poll_exit +2024-05-07 15:30:05,973 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-07 15:30:05,973 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: server_info +2024-05-07 15:30:05,974 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: poll_exit +2024-05-07 15:30:05,974 DEBUG SenderThread:7128 [sender.py:send_request():406] send_request: server_info +2024-05-07 15:30:05,974 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: get_summary +2024-05-07 15:30:05,975 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: sampled_history +2024-05-07 15:30:05,975 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-07 15:30:05,976 INFO MainThread:7128 [wandb_run.py:_footer_history_summary_info():3936] rendering history +2024-05-07 15:30:05,976 INFO MainThread:7128 [wandb_run.py:_footer_history_summary_info():3968] rendering summary +2024-05-07 15:30:05,976 DEBUG HandlerThread:7128 [handler.py:handle_request():146] handle_request: shutdown +2024-05-07 15:30:05,976 INFO HandlerThread:7128 [handler.py:finish():866] shutting down handler +2024-05-07 15:30:06,974 INFO WriterThread:7128 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240507_152956-gy8tfquj/run-gy8tfquj.wandb +2024-05-07 15:30:06,974 INFO SenderThread:7128 [sender.py:finish():1546] shutting down sender diff --git a/wandb/offline-run-20240507_152956-gy8tfquj/logs/debug.log b/wandb/offline-run-20240507_152956-gy8tfquj/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..a620080c228e11d649c5d372841a59824c54281e --- /dev/null +++ b/wandb/offline-run-20240507_152956-gy8tfquj/logs/debug.log @@ -0,0 +1,84 @@ +2024-05-07 15:29:56,393 INFO MainThread:6246 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_setup.py:_flush():76] Configure stats pid to 6246 +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240507_152956-gy8tfquj/logs/debug.log +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240507_152956-gy8tfquj/logs/debug-internal.log +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_init.py:init():561] calling init triggers +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_init.py:init():611] starting backend +2024-05-07 15:29:56,394 INFO MainThread:6246 [wandb_init.py:init():615] setting up manager +2024-05-07 15:29:56,399 INFO MainThread:6246 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-07 15:29:56,405 INFO MainThread:6246 [wandb_init.py:init():623] backend started and connected +2024-05-07 15:29:56,415 INFO MainThread:6246 [wandb_init.py:init():715] updated telemetry +2024-05-07 15:29:56,438 INFO MainThread:6246 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-07 15:29:56,446 INFO MainThread:6246 [wandb_init.py:init():799] starting run threads in backend +2024-05-07 15:30:05,894 WARNING MainThread:6246 [wandb_init.py:init():1203] interrupted +Traceback (most recent call last): + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/site-packages/wandb/sdk/wandb_init.py", line 1181, in init + run = wi.init() + ^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/site-packages/wandb/sdk/wandb_init.py", line 841, in init + run._on_start() + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/site-packages/wandb/sdk/wandb_run.py", line 2389, in _on_start + self._backend.interface.publish_python_packages(working_set()) + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/site-packages/wandb/sdk/interface/interface.py", line 295, in publish_python_packages + for pkg in working_set: + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/site-packages/wandb/util.py", line 1878, in working_set + yield InstalledDistribution(key=d.metadata["Name"], version=d.version) + ^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/importlib/metadata/__init__.py", line 611, in metadata + or self.read_text('PKG-INFO') + ^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/importlib/metadata/__init__.py", line 938, in read_text + return self._path.joinpath(filename).read_text(encoding='utf-8') + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/pathlib.py", line 1059, in read_text + return f.read() + ^^^^^^^^ + File "", line 319, in decode +KeyboardInterrupt + +During handling of the above exception, another exception occurred: + +Traceback (most recent call last): + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/site-packages/wandb/sdk/wandb_init.py", line 1189, in init + getcaller() + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/site-packages/wandb/sdk/wandb_init.py", line 849, in getcaller + src, line, func, stack = logger.findCaller(stack_info=True) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/logging/__init__.py", line 1589, in findCaller + traceback.print_stack(f, file=sio) + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/traceback.py", line 207, in print_stack + print_list(extract_stack(f, limit=limit), file=file) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/traceback.py", line 228, in extract_stack + stack = StackSummary.extract(walk_stack(f), limit=limit) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/traceback.py", line 390, in extract + return klass._extract_from_extended_frame_gen( + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/traceback.py", line 433, in _extract_from_extended_frame_gen + f.line + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/traceback.py", line 318, in line + self._line = linecache.getline(self.filename, self.lineno) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/linecache.py", line 30, in getline + lines = getlines(filename, module_globals) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/linecache.py", line 46, in getlines + return updatecache(filename, module_globals) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/linecache.py", line 136, in updatecache + with tokenize.open(fullname) as fp: + ^^^^^^^^^^^^^^^^^^^^^^^ + File "/scratch/groups/willhies/sschmidg/Anaconda3/lib/python3.11/tokenize.py", line 396, in open + buffer = _builtin_open(filename, 'rb') + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +KeyboardInterrupt diff --git a/wandb/offline-run-20240507_152956-gy8tfquj/run-gy8tfquj.wandb b/wandb/offline-run-20240507_152956-gy8tfquj/run-gy8tfquj.wandb new file mode 100644 index 0000000000000000000000000000000000000000..31e53db3787ed697eddbfb51a720fc72ea00e424 Binary files /dev/null and b/wandb/offline-run-20240507_152956-gy8tfquj/run-gy8tfquj.wandb differ diff --git a/wandb/offline-run-20240508_185440-w4yruoqg/files/conda-environment.yaml b/wandb/offline-run-20240508_185440-w4yruoqg/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ab80d91aabc2199c787d0c4d29ff1e49b7f1807 --- /dev/null +++ b/wandb/offline-run-20240508_185440-w4yruoqg/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.31.0 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240508_185440-w4yruoqg/files/wandb-metadata.json b/wandb/offline-run-20240508_185440-w4yruoqg/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..ddf7400df44c367d1025d6aeed19a02b8583d69c --- /dev/null +++ b/wandb/offline-run-20240508_185440-w4yruoqg/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T01:54:40.514231", + "startedAt": "2024-05-09T01:54:40.337963", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.6, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.2, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.3213043212890625 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240508_185440-w4yruoqg/logs/debug-internal.log b/wandb/offline-run-20240508_185440-w4yruoqg/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5424f350ad95da5a549a7b66d7a486af3ce70b20 --- /dev/null +++ b/wandb/offline-run-20240508_185440-w4yruoqg/logs/debug-internal.log @@ -0,0 +1,40 @@ +2024-05-08 18:54:40,364 INFO StreamThr :5228 [internal.py:wandb_internal():86] W&B internal server running at pid: 5228, started at: 2024-05-08 18:54:40.362073 +2024-05-08 18:54:40,367 DEBUG HandlerThread:5228 [handler.py:handle_request():146] handle_request: status +2024-05-08 18:54:40,376 INFO WriterThread:5228 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_185440-w4yruoqg/run-w4yruoqg.wandb +2024-05-08 18:54:40,456 DEBUG HandlerThread:5228 [handler.py:handle_request():146] handle_request: run_start +2024-05-08 18:54:40,478 DEBUG HandlerThread:5228 [system_info.py:__init__():26] System info init +2024-05-08 18:54:40,478 DEBUG HandlerThread:5228 [system_info.py:__init__():41] System info init done +2024-05-08 18:54:40,478 INFO HandlerThread:5228 [system_monitor.py:start():194] Starting system monitor +2024-05-08 18:54:40,479 INFO SystemMonitor:5228 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-08 18:54:40,479 INFO HandlerThread:5228 [system_monitor.py:probe():214] Collecting system info +2024-05-08 18:54:40,480 INFO SystemMonitor:5228 [interfaces.py:start():190] Started cpu monitoring +2024-05-08 18:54:40,481 INFO SystemMonitor:5228 [interfaces.py:start():190] Started disk monitoring +2024-05-08 18:54:40,482 INFO SystemMonitor:5228 [interfaces.py:start():190] Started gpu monitoring +2024-05-08 18:54:40,483 INFO SystemMonitor:5228 [interfaces.py:start():190] Started memory monitoring +2024-05-08 18:54:40,484 INFO SystemMonitor:5228 [interfaces.py:start():190] Started network monitoring +2024-05-08 18:54:40,514 DEBUG HandlerThread:5228 [system_info.py:probe():150] Probing system +2024-05-08 18:54:40,517 DEBUG HandlerThread:5228 [system_info.py:_probe_git():135] Probing git +2024-05-08 18:54:40,530 DEBUG HandlerThread:5228 [system_info.py:_probe_git():143] Probing git done +2024-05-08 18:54:40,530 DEBUG HandlerThread:5228 [system_info.py:probe():198] Probing system done +2024-05-08 18:54:40,531 DEBUG HandlerThread:5228 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T01:54:40.514231', 'startedAt': '2024-05-09T01:54:40.337963', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.6, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.2, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.3213043212890625}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-08 18:54:40,531 INFO HandlerThread:5228 [system_monitor.py:probe():224] Finished collecting system info +2024-05-08 18:54:40,531 INFO HandlerThread:5228 [system_monitor.py:probe():227] Publishing system info +2024-05-08 18:54:40,531 DEBUG HandlerThread:5228 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-08 18:54:50,872 DEBUG HandlerThread:5228 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-08 18:54:50,875 INFO HandlerThread:5228 [system_monitor.py:probe():229] Finished publishing system info +2024-05-08 18:54:50,887 DEBUG HandlerThread:5228 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 18:54:50,887 DEBUG HandlerThread:5228 [handler.py:handle_request():146] handle_request: keepalive +2024-05-08 18:54:50,888 DEBUG HandlerThread:5228 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 18:54:50,888 DEBUG HandlerThread:5228 [handler.py:handle_request():146] handle_request: keepalive +2024-05-08 18:54:50,888 DEBUG SenderThread:5228 [sender.py:send_request():406] send_request: status_report +2024-05-08 18:54:50,889 DEBUG SenderThread:5228 [sender.py:send_request():406] send_request: status_report +2024-05-08 18:54:52,105 DEBUG HandlerThread:5228 [handler.py:handle_request():146] handle_request: python_packages +2024-05-08 18:54:55,889 DEBUG HandlerThread:5228 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 18:54:55,890 DEBUG SenderThread:5228 [sender.py:send_request():406] send_request: status_report +2024-05-08 18:54:56,496 INFO memory :5228 [interfaces.py:monitor():140] Process proc.memory.rssMB has exited. +2024-05-08 18:54:56,497 DEBUG SystemMonitor:5228 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-08 18:54:56,498 DEBUG SystemMonitor:5228 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-08 18:54:56,498 DEBUG SystemMonitor:5228 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-08 18:54:57,369 WARNING StreamThr :5228 [internal.py:is_dead():414] Internal process exiting, parent pid 4912 disappeared +2024-05-08 18:54:57,369 ERROR StreamThr :5228 [internal.py:wandb_internal():152] Internal process shutdown. +2024-05-08 18:54:57,422 INFO MainThread:5228 [internal.py:handle_exit():76] Internal process exited diff --git a/wandb/offline-run-20240508_185440-w4yruoqg/logs/debug.log b/wandb/offline-run-20240508_185440-w4yruoqg/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..596e553a071508d63347b619349d47713f9bb148 --- /dev/null +++ b/wandb/offline-run-20240508_185440-w4yruoqg/logs/debug.log @@ -0,0 +1,26 @@ +2024-05-08 18:54:40,349 INFO MainThread:4912 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-08 18:54:40,350 INFO MainThread:4912 [wandb_setup.py:_flush():76] Configure stats pid to 4912 +2024-05-08 18:54:40,350 INFO MainThread:4912 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-08 18:54:40,350 INFO MainThread:4912 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-08 18:54:40,350 INFO MainThread:4912 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-08 18:54:40,350 INFO MainThread:4912 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-08 18:54:40,350 INFO MainThread:4912 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-08 18:54:40,350 INFO MainThread:4912 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-08 18:54:40,351 INFO MainThread:4912 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-08 18:54:40,351 INFO MainThread:4912 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_185440-w4yruoqg/logs/debug.log +2024-05-08 18:54:40,351 INFO MainThread:4912 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_185440-w4yruoqg/logs/debug-internal.log +2024-05-08 18:54:40,351 INFO MainThread:4912 [wandb_init.py:init():561] calling init triggers +2024-05-08 18:54:40,351 INFO MainThread:4912 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-08 18:54:40,351 INFO MainThread:4912 [wandb_init.py:init():611] starting backend +2024-05-08 18:54:40,351 INFO MainThread:4912 [wandb_init.py:init():615] setting up manager +2024-05-08 18:54:40,360 INFO MainThread:4912 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-08 18:54:40,366 INFO MainThread:4912 [wandb_init.py:init():623] backend started and connected +2024-05-08 18:54:40,376 INFO MainThread:4912 [wandb_init.py:init():715] updated telemetry +2024-05-08 18:54:40,403 INFO MainThread:4912 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-08 18:54:40,411 INFO MainThread:4912 [wandb_init.py:init():799] starting run threads in backend +2024-05-08 18:54:52,104 INFO MainThread:4912 [wandb_run.py:_console_start():2335] atexit reg +2024-05-08 18:54:52,104 INFO MainThread:4912 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-08 18:54:52,104 INFO MainThread:4912 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-08 18:54:52,104 INFO MainThread:4912 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-08 18:54:52,105 INFO MainThread:4912 [wandb_init.py:init():842] run started, returning control to user process diff --git a/wandb/offline-run-20240508_185440-w4yruoqg/run-w4yruoqg.wandb b/wandb/offline-run-20240508_185440-w4yruoqg/run-w4yruoqg.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/wandb/offline-run-20240508_190127-u0qvu51w/files/conda-environment.yaml b/wandb/offline-run-20240508_190127-u0qvu51w/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ab80d91aabc2199c787d0c4d29ff1e49b7f1807 --- /dev/null +++ b/wandb/offline-run-20240508_190127-u0qvu51w/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.31.0 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240508_190127-u0qvu51w/files/wandb-metadata.json b/wandb/offline-run-20240508_190127-u0qvu51w/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b0f81f6970a12ea5d22020990f29d9519b14f1f0 --- /dev/null +++ b/wandb/offline-run-20240508_190127-u0qvu51w/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T02:01:27.315057", + "startedAt": "2024-05-09T02:01:27.154285", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.5625, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.3232879638671875 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240508_190127-u0qvu51w/logs/debug-internal.log b/wandb/offline-run-20240508_190127-u0qvu51w/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..68737a5cd1d947db0a9b769001a458d39c6dcc9f --- /dev/null +++ b/wandb/offline-run-20240508_190127-u0qvu51w/logs/debug-internal.log @@ -0,0 +1,46 @@ +2024-05-08 19:01:27,179 INFO StreamThr :6496 [internal.py:wandb_internal():86] W&B internal server running at pid: 6496, started at: 2024-05-08 19:01:27.176652 +2024-05-08 19:01:27,182 DEBUG HandlerThread:6496 [handler.py:handle_request():146] handle_request: status +2024-05-08 19:01:27,191 INFO WriterThread:6496 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_190127-u0qvu51w/run-u0qvu51w.wandb +2024-05-08 19:01:27,253 DEBUG HandlerThread:6496 [handler.py:handle_request():146] handle_request: run_start +2024-05-08 19:01:27,271 DEBUG HandlerThread:6496 [system_info.py:__init__():26] System info init +2024-05-08 19:01:27,271 DEBUG HandlerThread:6496 [system_info.py:__init__():41] System info init done +2024-05-08 19:01:27,271 INFO HandlerThread:6496 [system_monitor.py:start():194] Starting system monitor +2024-05-08 19:01:27,271 INFO SystemMonitor:6496 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-08 19:01:27,271 INFO HandlerThread:6496 [system_monitor.py:probe():214] Collecting system info +2024-05-08 19:01:27,284 INFO SystemMonitor:6496 [interfaces.py:start():190] Started cpu monitoring +2024-05-08 19:01:27,285 INFO SystemMonitor:6496 [interfaces.py:start():190] Started disk monitoring +2024-05-08 19:01:27,285 INFO SystemMonitor:6496 [interfaces.py:start():190] Started gpu monitoring +2024-05-08 19:01:27,286 INFO SystemMonitor:6496 [interfaces.py:start():190] Started memory monitoring +2024-05-08 19:01:27,287 INFO SystemMonitor:6496 [interfaces.py:start():190] Started network monitoring +2024-05-08 19:01:27,314 DEBUG HandlerThread:6496 [system_info.py:probe():150] Probing system +2024-05-08 19:01:27,318 DEBUG HandlerThread:6496 [system_info.py:_probe_git():135] Probing git +2024-05-08 19:01:27,331 DEBUG HandlerThread:6496 [system_info.py:_probe_git():143] Probing git done +2024-05-08 19:01:27,331 DEBUG HandlerThread:6496 [system_info.py:probe():198] Probing system done +2024-05-08 19:01:27,331 DEBUG HandlerThread:6496 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T02:01:27.315057', 'startedAt': '2024-05-09T02:01:27.154285', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.5625, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.3232879638671875}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-08 19:01:27,331 INFO HandlerThread:6496 [system_monitor.py:probe():224] Finished collecting system info +2024-05-08 19:01:27,331 INFO HandlerThread:6496 [system_monitor.py:probe():227] Publishing system info +2024-05-08 19:01:27,331 DEBUG HandlerThread:6496 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-08 19:01:35,463 DEBUG HandlerThread:6496 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-08 19:01:35,467 INFO HandlerThread:6496 [system_monitor.py:probe():229] Finished publishing system info +2024-05-08 19:01:35,472 DEBUG HandlerThread:6496 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:01:35,472 DEBUG HandlerThread:6496 [handler.py:handle_request():146] handle_request: keepalive +2024-05-08 19:01:35,472 DEBUG SenderThread:6496 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:01:35,752 DEBUG HandlerThread:6496 [handler.py:handle_request():146] handle_request: python_packages +2024-05-08 19:01:37,473 DEBUG HandlerThread:6496 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:01:37,474 DEBUG SenderThread:6496 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:01:42,475 DEBUG HandlerThread:6496 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:01:42,475 DEBUG SenderThread:6496 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:01:44,184 WARNING StreamThr :6496 [internal.py:is_dead():414] Internal process exiting, parent pid 6220 disappeared +2024-05-08 19:01:44,184 ERROR StreamThr :6496 [internal.py:wandb_internal():152] Internal process shutdown. +2024-05-08 19:01:44,476 INFO SenderThread:6496 [sender.py:finish():1546] shutting down sender +2024-05-08 19:01:45,113 INFO HandlerThread:6496 [handler.py:finish():866] shutting down handler +2024-05-08 19:01:45,113 INFO WriterThread:6496 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_190127-u0qvu51w/run-u0qvu51w.wandb +2024-05-08 19:01:45,113 INFO HandlerThread:6496 [system_monitor.py:finish():203] Stopping system monitor +2024-05-08 19:01:45,113 DEBUG SystemMonitor:6496 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-08 19:01:45,114 INFO HandlerThread:6496 [interfaces.py:finish():202] Joined cpu monitor +2024-05-08 19:01:45,114 DEBUG SystemMonitor:6496 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-08 19:01:45,115 INFO HandlerThread:6496 [interfaces.py:finish():202] Joined disk monitor +2024-05-08 19:01:45,115 DEBUG SystemMonitor:6496 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-08 19:01:45,118 INFO HandlerThread:6496 [interfaces.py:finish():202] Joined gpu monitor +2024-05-08 19:01:45,118 INFO HandlerThread:6496 [interfaces.py:finish():202] Joined memory monitor +2024-05-08 19:01:45,119 INFO HandlerThread:6496 [interfaces.py:finish():202] Joined network monitor diff --git a/wandb/offline-run-20240508_190127-u0qvu51w/logs/debug.log b/wandb/offline-run-20240508_190127-u0qvu51w/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0750eeff14c21e290a3060fa9cd24ef8ebb266a6 --- /dev/null +++ b/wandb/offline-run-20240508_190127-u0qvu51w/logs/debug.log @@ -0,0 +1,26 @@ +2024-05-08 19:01:27,166 INFO MainThread:6220 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-08 19:01:27,167 INFO MainThread:6220 [wandb_setup.py:_flush():76] Configure stats pid to 6220 +2024-05-08 19:01:27,167 INFO MainThread:6220 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_190127-u0qvu51w/logs/debug.log +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_190127-u0qvu51w/logs/debug-internal.log +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_init.py:init():561] calling init triggers +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_init.py:init():611] starting backend +2024-05-08 19:01:27,168 INFO MainThread:6220 [wandb_init.py:init():615] setting up manager +2024-05-08 19:01:27,174 INFO MainThread:6220 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-08 19:01:27,178 INFO MainThread:6220 [wandb_init.py:init():623] backend started and connected +2024-05-08 19:01:27,190 INFO MainThread:6220 [wandb_init.py:init():715] updated telemetry +2024-05-08 19:01:27,202 INFO MainThread:6220 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-08 19:01:27,209 INFO MainThread:6220 [wandb_init.py:init():799] starting run threads in backend +2024-05-08 19:01:35,751 INFO MainThread:6220 [wandb_run.py:_console_start():2335] atexit reg +2024-05-08 19:01:35,751 INFO MainThread:6220 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-08 19:01:35,751 INFO MainThread:6220 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-08 19:01:35,751 INFO MainThread:6220 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-08 19:01:35,752 INFO MainThread:6220 [wandb_init.py:init():842] run started, returning control to user process diff --git a/wandb/offline-run-20240508_190127-u0qvu51w/run-u0qvu51w.wandb b/wandb/offline-run-20240508_190127-u0qvu51w/run-u0qvu51w.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b4f282b16a8b83758e0da72a5bee16eec9249f8a Binary files /dev/null and b/wandb/offline-run-20240508_190127-u0qvu51w/run-u0qvu51w.wandb differ diff --git a/wandb/offline-run-20240508_191952-7qm3st83/files/conda-environment.yaml b/wandb/offline-run-20240508_191952-7qm3st83/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ab80d91aabc2199c787d0c4d29ff1e49b7f1807 --- /dev/null +++ b/wandb/offline-run-20240508_191952-7qm3st83/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.31.0 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240508_191952-7qm3st83/files/wandb-metadata.json b/wandb/offline-run-20240508_191952-7qm3st83/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e4041aebdc3f48b800a40a2e1403a34040f4948e --- /dev/null +++ b/wandb/offline-run-20240508_191952-7qm3st83/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T02:19:52.292648", + "startedAt": "2024-05-09T02:19:52.140435", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.56875, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.2, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.2, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.321495056152344 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240508_191952-7qm3st83/logs/debug-internal.log b/wandb/offline-run-20240508_191952-7qm3st83/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..5f702db7ba891909d2673f019b6b2390a6733630 --- /dev/null +++ b/wandb/offline-run-20240508_191952-7qm3st83/logs/debug-internal.log @@ -0,0 +1,40 @@ +2024-05-08 19:19:52,162 INFO StreamThr :8980 [internal.py:wandb_internal():86] W&B internal server running at pid: 8980, started at: 2024-05-08 19:19:52.160108 +2024-05-08 19:19:52,165 DEBUG HandlerThread:8980 [handler.py:handle_request():146] handle_request: status +2024-05-08 19:19:52,170 INFO WriterThread:8980 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_191952-7qm3st83/run-7qm3st83.wandb +2024-05-08 19:19:52,230 DEBUG HandlerThread:8980 [handler.py:handle_request():146] handle_request: run_start +2024-05-08 19:19:52,256 DEBUG HandlerThread:8980 [system_info.py:__init__():26] System info init +2024-05-08 19:19:52,256 DEBUG HandlerThread:8980 [system_info.py:__init__():41] System info init done +2024-05-08 19:19:52,257 INFO HandlerThread:8980 [system_monitor.py:start():194] Starting system monitor +2024-05-08 19:19:52,257 INFO SystemMonitor:8980 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-08 19:19:52,257 INFO HandlerThread:8980 [system_monitor.py:probe():214] Collecting system info +2024-05-08 19:19:52,258 INFO SystemMonitor:8980 [interfaces.py:start():190] Started cpu monitoring +2024-05-08 19:19:52,259 INFO SystemMonitor:8980 [interfaces.py:start():190] Started disk monitoring +2024-05-08 19:19:52,261 INFO SystemMonitor:8980 [interfaces.py:start():190] Started gpu monitoring +2024-05-08 19:19:52,261 INFO SystemMonitor:8980 [interfaces.py:start():190] Started memory monitoring +2024-05-08 19:19:52,262 INFO SystemMonitor:8980 [interfaces.py:start():190] Started network monitoring +2024-05-08 19:19:52,292 DEBUG HandlerThread:8980 [system_info.py:probe():150] Probing system +2024-05-08 19:19:52,295 DEBUG HandlerThread:8980 [system_info.py:_probe_git():135] Probing git +2024-05-08 19:19:52,306 DEBUG HandlerThread:8980 [system_info.py:_probe_git():143] Probing git done +2024-05-08 19:19:52,307 DEBUG HandlerThread:8980 [system_info.py:probe():198] Probing system done +2024-05-08 19:19:52,307 DEBUG HandlerThread:8980 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T02:19:52.292648', 'startedAt': '2024-05-09T02:19:52.140435', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.56875, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.2, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.2, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.321495056152344}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-08 19:19:52,307 INFO HandlerThread:8980 [system_monitor.py:probe():224] Finished collecting system info +2024-05-08 19:19:52,307 INFO HandlerThread:8980 [system_monitor.py:probe():227] Publishing system info +2024-05-08 19:19:52,307 DEBUG HandlerThread:8980 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-08 19:20:00,539 DEBUG HandlerThread:8980 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-08 19:20:00,546 INFO HandlerThread:8980 [system_monitor.py:probe():229] Finished publishing system info +2024-05-08 19:20:00,551 DEBUG HandlerThread:8980 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:20:00,552 DEBUG HandlerThread:8980 [handler.py:handle_request():146] handle_request: keepalive +2024-05-08 19:20:00,552 DEBUG SenderThread:8980 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:20:00,843 DEBUG HandlerThread:8980 [handler.py:handle_request():146] handle_request: python_packages +2024-05-08 19:20:02,553 DEBUG HandlerThread:8980 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:20:02,553 DEBUG SenderThread:8980 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:20:07,554 DEBUG HandlerThread:8980 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:20:07,555 DEBUG SenderThread:8980 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:20:08,276 INFO cpu :8980 [interfaces.py:monitor():140] Process cpu has exited. +2024-05-08 19:20:08,277 DEBUG SystemMonitor:8980 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-08 19:20:08,277 DEBUG SystemMonitor:8980 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-08 19:20:08,277 DEBUG SystemMonitor:8980 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-08 19:20:08,280 INFO memory :8980 [interfaces.py:monitor():140] Process proc.memory.rssMB has exited. +2024-05-08 19:20:09,167 WARNING StreamThr :8980 [internal.py:is_dead():414] Internal process exiting, parent pid 8723 disappeared +2024-05-08 19:20:09,167 ERROR StreamThr :8980 [internal.py:wandb_internal():152] Internal process shutdown. +2024-05-08 19:20:09,195 INFO MainThread:8980 [internal.py:handle_exit():76] Internal process exited diff --git a/wandb/offline-run-20240508_191952-7qm3st83/logs/debug.log b/wandb/offline-run-20240508_191952-7qm3st83/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..d73f3d35586d780b045d924ce968aa8bc458a798 --- /dev/null +++ b/wandb/offline-run-20240508_191952-7qm3st83/logs/debug.log @@ -0,0 +1,26 @@ +2024-05-08 19:19:52,151 INFO MainThread:8723 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-08 19:19:52,152 INFO MainThread:8723 [wandb_setup.py:_flush():76] Configure stats pid to 8723 +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_191952-7qm3st83/logs/debug.log +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_191952-7qm3st83/logs/debug-internal.log +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_init.py:init():561] calling init triggers +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_init.py:init():611] starting backend +2024-05-08 19:19:52,153 INFO MainThread:8723 [wandb_init.py:init():615] setting up manager +2024-05-08 19:19:52,158 INFO MainThread:8723 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-08 19:19:52,160 INFO MainThread:8723 [wandb_init.py:init():623] backend started and connected +2024-05-08 19:19:52,170 INFO MainThread:8723 [wandb_init.py:init():715] updated telemetry +2024-05-08 19:19:52,181 INFO MainThread:8723 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-08 19:19:52,186 INFO MainThread:8723 [wandb_init.py:init():799] starting run threads in backend +2024-05-08 19:20:00,843 INFO MainThread:8723 [wandb_run.py:_console_start():2335] atexit reg +2024-05-08 19:20:00,843 INFO MainThread:8723 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-08 19:20:00,843 INFO MainThread:8723 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-08 19:20:00,843 INFO MainThread:8723 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-08 19:20:00,844 INFO MainThread:8723 [wandb_init.py:init():842] run started, returning control to user process diff --git a/wandb/offline-run-20240508_191952-7qm3st83/run-7qm3st83.wandb b/wandb/offline-run-20240508_191952-7qm3st83/run-7qm3st83.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/wandb/offline-run-20240508_192346-bmqnwzmb/files/conda-environment.yaml b/wandb/offline-run-20240508_192346-bmqnwzmb/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ed56ab5ac90a4bed6f79ed5c79ec57e153ec12f --- /dev/null +++ b/wandb/offline-run-20240508_192346-bmqnwzmb/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.37.2 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240508_192346-bmqnwzmb/files/wandb-metadata.json b/wandb/offline-run-20240508_192346-bmqnwzmb/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..e99df72b35882fd19c1e867a4e2b05c47e15c5e7 --- /dev/null +++ b/wandb/offline-run-20240508_192346-bmqnwzmb/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T02:23:46.697731", + "startedAt": "2024-05-09T02:23:46.539171", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.56875, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.2, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.321506500244141 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240508_192346-bmqnwzmb/files/wandb-summary.json b/wandb/offline-run-20240508_192346-bmqnwzmb/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..30fe69f73261d9403d3ab1103903e365d1dc3a2f --- /dev/null +++ b/wandb/offline-run-20240508_192346-bmqnwzmb/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 20}} \ No newline at end of file diff --git a/wandb/offline-run-20240508_192346-bmqnwzmb/logs/debug-internal.log b/wandb/offline-run-20240508_192346-bmqnwzmb/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..09d53cc4e33bfc4a65c174d3d2151c9c985e2e84 --- /dev/null +++ b/wandb/offline-run-20240508_192346-bmqnwzmb/logs/debug-internal.log @@ -0,0 +1,140 @@ +2024-05-08 19:23:46,562 INFO StreamThr :9792 [internal.py:wandb_internal():86] W&B internal server running at pid: 9792, started at: 2024-05-08 19:23:46.560027 +2024-05-08 19:23:46,565 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: status +2024-05-08 19:23:46,572 INFO WriterThread:9792 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_192346-bmqnwzmb/run-bmqnwzmb.wandb +2024-05-08 19:23:46,636 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: run_start +2024-05-08 19:23:46,661 DEBUG HandlerThread:9792 [system_info.py:__init__():26] System info init +2024-05-08 19:23:46,662 DEBUG HandlerThread:9792 [system_info.py:__init__():41] System info init done +2024-05-08 19:23:46,662 INFO HandlerThread:9792 [system_monitor.py:start():194] Starting system monitor +2024-05-08 19:23:46,662 INFO SystemMonitor:9792 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-08 19:23:46,662 INFO HandlerThread:9792 [system_monitor.py:probe():214] Collecting system info +2024-05-08 19:23:46,663 INFO SystemMonitor:9792 [interfaces.py:start():190] Started cpu monitoring +2024-05-08 19:23:46,664 INFO SystemMonitor:9792 [interfaces.py:start():190] Started disk monitoring +2024-05-08 19:23:46,665 INFO SystemMonitor:9792 [interfaces.py:start():190] Started gpu monitoring +2024-05-08 19:23:46,666 INFO SystemMonitor:9792 [interfaces.py:start():190] Started memory monitoring +2024-05-08 19:23:46,669 INFO SystemMonitor:9792 [interfaces.py:start():190] Started network monitoring +2024-05-08 19:23:46,697 DEBUG HandlerThread:9792 [system_info.py:probe():150] Probing system +2024-05-08 19:23:46,700 DEBUG HandlerThread:9792 [system_info.py:_probe_git():135] Probing git +2024-05-08 19:23:46,712 DEBUG HandlerThread:9792 [system_info.py:_probe_git():143] Probing git done +2024-05-08 19:23:46,712 DEBUG HandlerThread:9792 [system_info.py:probe():198] Probing system done +2024-05-08 19:23:46,712 DEBUG HandlerThread:9792 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T02:23:46.697731', 'startedAt': '2024-05-09T02:23:46.539171', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.56875, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.2, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.321506500244141}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-08 19:23:46,713 INFO HandlerThread:9792 [system_monitor.py:probe():224] Finished collecting system info +2024-05-08 19:23:46,713 INFO HandlerThread:9792 [system_monitor.py:probe():227] Publishing system info +2024-05-08 19:23:46,713 DEBUG HandlerThread:9792 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-08 19:23:54,845 DEBUG HandlerThread:9792 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-08 19:23:54,849 INFO HandlerThread:9792 [system_monitor.py:probe():229] Finished publishing system info +2024-05-08 19:23:54,853 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:23:54,853 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: keepalive +2024-05-08 19:23:54,854 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:23:55,120 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: python_packages +2024-05-08 19:23:56,855 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:23:56,855 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:24:01,856 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:24:01,857 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:24:06,858 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:24:06,858 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:24:06,922 DEBUG SenderThread:9792 [sender.py:send():379] send: exit +2024-05-08 19:24:06,922 INFO SenderThread:9792 [sender.py:send_exit():586] handling exit code: 1 +2024-05-08 19:24:06,922 INFO SenderThread:9792 [sender.py:send_exit():588] handling runtime: 20 +2024-05-08 19:24:06,925 INFO SenderThread:9792 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-08 19:24:06,925 INFO SenderThread:9792 [sender.py:send_exit():594] send defer +2024-05-08 19:24:06,925 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,925 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 0 +2024-05-08 19:24:06,925 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,925 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-05-08 19:24:06,925 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 1 +2024-05-08 19:24:06,926 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,926 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 1 +2024-05-08 19:24:06,926 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,926 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-05-08 19:24:06,926 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 2 +2024-05-08 19:24:06,926 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,926 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 2 +2024-05-08 19:24:06,926 INFO HandlerThread:9792 [system_monitor.py:finish():203] Stopping system monitor +2024-05-08 19:24:06,926 DEBUG SystemMonitor:9792 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-08 19:24:06,927 DEBUG SystemMonitor:9792 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-08 19:24:06,927 DEBUG SystemMonitor:9792 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-08 19:24:06,927 INFO HandlerThread:9792 [interfaces.py:finish():202] Joined cpu monitor +2024-05-08 19:24:06,929 INFO HandlerThread:9792 [interfaces.py:finish():202] Joined disk monitor +2024-05-08 19:24:06,962 INFO HandlerThread:9792 [interfaces.py:finish():202] Joined gpu monitor +2024-05-08 19:24:06,962 INFO HandlerThread:9792 [interfaces.py:finish():202] Joined memory monitor +2024-05-08 19:24:06,962 INFO HandlerThread:9792 [interfaces.py:finish():202] Joined network monitor +2024-05-08 19:24:06,963 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,963 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-05-08 19:24:06,963 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 3 +2024-05-08 19:24:06,963 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,963 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 3 +2024-05-08 19:24:06,964 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,964 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-05-08 19:24:06,964 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 4 +2024-05-08 19:24:06,964 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,964 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 4 +2024-05-08 19:24:06,964 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,964 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-05-08 19:24:06,964 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 5 +2024-05-08 19:24:06,964 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,965 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 5 +2024-05-08 19:24:06,965 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,965 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-05-08 19:24:06,965 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 6 +2024-05-08 19:24:06,965 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,965 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 6 +2024-05-08 19:24:06,965 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,965 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-05-08 19:24:06,965 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 7 +2024-05-08 19:24:06,966 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:24:06,966 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,966 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 7 +2024-05-08 19:24:06,966 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:24:06,966 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,966 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-05-08 19:24:06,966 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 8 +2024-05-08 19:24:06,966 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,966 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 8 +2024-05-08 19:24:06,967 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,967 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-05-08 19:24:06,967 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 9 +2024-05-08 19:24:06,967 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,967 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 9 +2024-05-08 19:24:06,967 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,967 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-05-08 19:24:06,967 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 10 +2024-05-08 19:24:06,967 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,967 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 10 +2024-05-08 19:24:06,967 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,968 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-05-08 19:24:06,968 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 11 +2024-05-08 19:24:06,968 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,968 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 11 +2024-05-08 19:24:06,968 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,968 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-05-08 19:24:06,968 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 12 +2024-05-08 19:24:06,968 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,968 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 12 +2024-05-08 19:24:06,968 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,968 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-05-08 19:24:06,969 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 13 +2024-05-08 19:24:06,969 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,969 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 13 +2024-05-08 19:24:06,969 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,969 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-05-08 19:24:06,969 INFO SenderThread:9792 [sender.py:transition_state():614] send defer: 14 +2024-05-08 19:24:06,969 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: defer +2024-05-08 19:24:06,969 DEBUG SenderThread:9792 [sender.py:send():379] send: final +2024-05-08 19:24:06,969 INFO HandlerThread:9792 [handler.py:handle_request_defer():172] handle defer: 14 +2024-05-08 19:24:06,970 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: defer +2024-05-08 19:24:06,970 INFO SenderThread:9792 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-05-08 19:24:06,970 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-08 19:24:06,970 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: poll_exit +2024-05-08 19:24:06,971 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-08 19:24:06,971 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: poll_exit +2024-05-08 19:24:06,971 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: server_info +2024-05-08 19:24:06,972 DEBUG SenderThread:9792 [sender.py:send_request():406] send_request: server_info +2024-05-08 19:24:06,972 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: get_summary +2024-05-08 19:24:06,972 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: sampled_history +2024-05-08 19:24:06,973 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-08 19:24:06,973 INFO MainThread:9792 [wandb_run.py:_footer_history_summary_info():3936] rendering history +2024-05-08 19:24:06,973 INFO MainThread:9792 [wandb_run.py:_footer_history_summary_info():3968] rendering summary +2024-05-08 19:24:06,973 DEBUG HandlerThread:9792 [handler.py:handle_request():146] handle_request: shutdown +2024-05-08 19:24:06,973 INFO HandlerThread:9792 [handler.py:finish():866] shutting down handler +2024-05-08 19:24:07,972 INFO WriterThread:9792 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_192346-bmqnwzmb/run-bmqnwzmb.wandb +2024-05-08 19:24:07,972 INFO SenderThread:9792 [sender.py:finish():1546] shutting down sender diff --git a/wandb/offline-run-20240508_192346-bmqnwzmb/logs/debug.log b/wandb/offline-run-20240508_192346-bmqnwzmb/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..3250daa6fe334ff1544e833395358ab11aaa0544 --- /dev/null +++ b/wandb/offline-run-20240508_192346-bmqnwzmb/logs/debug.log @@ -0,0 +1,27 @@ +2024-05-08 19:23:46,551 INFO MainThread:9572 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-08 19:23:46,552 INFO MainThread:9572 [wandb_setup.py:_flush():76] Configure stats pid to 9572 +2024-05-08 19:23:46,552 INFO MainThread:9572 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-08 19:23:46,552 INFO MainThread:9572 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-08 19:23:46,552 INFO MainThread:9572 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-08 19:23:46,552 INFO MainThread:9572 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-08 19:23:46,552 INFO MainThread:9572 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-08 19:23:46,552 INFO MainThread:9572 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-08 19:23:46,552 INFO MainThread:9572 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-08 19:23:46,552 INFO MainThread:9572 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_192346-bmqnwzmb/logs/debug.log +2024-05-08 19:23:46,553 INFO MainThread:9572 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_192346-bmqnwzmb/logs/debug-internal.log +2024-05-08 19:23:46,553 INFO MainThread:9572 [wandb_init.py:init():561] calling init triggers +2024-05-08 19:23:46,553 INFO MainThread:9572 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-08 19:23:46,553 INFO MainThread:9572 [wandb_init.py:init():611] starting backend +2024-05-08 19:23:46,553 INFO MainThread:9572 [wandb_init.py:init():615] setting up manager +2024-05-08 19:23:46,558 INFO MainThread:9572 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-08 19:23:46,561 INFO MainThread:9572 [wandb_init.py:init():623] backend started and connected +2024-05-08 19:23:46,572 INFO MainThread:9572 [wandb_init.py:init():715] updated telemetry +2024-05-08 19:23:46,583 INFO MainThread:9572 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-08 19:23:46,590 INFO MainThread:9572 [wandb_init.py:init():799] starting run threads in backend +2024-05-08 19:23:55,119 INFO MainThread:9572 [wandb_run.py:_console_start():2335] atexit reg +2024-05-08 19:23:55,119 INFO MainThread:9572 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-08 19:23:55,119 INFO MainThread:9572 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-08 19:23:55,119 INFO MainThread:9572 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-08 19:23:55,120 INFO MainThread:9572 [wandb_init.py:init():842] run started, returning control to user process +2024-05-08 19:24:07,975 WARNING MsgRouterThr:9572 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/offline-run-20240508_192346-bmqnwzmb/run-bmqnwzmb.wandb b/wandb/offline-run-20240508_192346-bmqnwzmb/run-bmqnwzmb.wandb new file mode 100644 index 0000000000000000000000000000000000000000..89a737ff433c0b58ef4b34becd3cd3e7c9a125dc Binary files /dev/null and b/wandb/offline-run-20240508_192346-bmqnwzmb/run-bmqnwzmb.wandb differ diff --git a/wandb/offline-run-20240508_195258-zu5yru3s/files/conda-environment.yaml b/wandb/offline-run-20240508_195258-zu5yru3s/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb5c46cd8a524560371e6601fcdbf9ef63efe33a --- /dev/null +++ b/wandb/offline-run-20240508_195258-zu5yru3s/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.40.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240508_195258-zu5yru3s/files/wandb-metadata.json b/wandb/offline-run-20240508_195258-zu5yru3s/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..8e71c2ec1b3e82173d03d1e23a307c5552a5f1f4 --- /dev/null +++ b/wandb/offline-run-20240508_195258-zu5yru3s/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T02:52:58.246583", + "startedAt": "2024-05-09T02:52:58.086152", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.56875, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.321647644042969 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240508_195258-zu5yru3s/logs/debug-internal.log b/wandb/offline-run-20240508_195258-zu5yru3s/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4147b25ccc8a2793eb13808e17ed6e6aefeb89f7 --- /dev/null +++ b/wandb/offline-run-20240508_195258-zu5yru3s/logs/debug-internal.log @@ -0,0 +1,42 @@ +2024-05-08 19:52:58,110 INFO StreamThr :12969 [internal.py:wandb_internal():86] W&B internal server running at pid: 12969, started at: 2024-05-08 19:52:58.107939 +2024-05-08 19:52:58,113 DEBUG HandlerThread:12969 [handler.py:handle_request():146] handle_request: status +2024-05-08 19:52:58,122 INFO WriterThread:12969 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_195258-zu5yru3s/run-zu5yru3s.wandb +2024-05-08 19:52:58,186 DEBUG HandlerThread:12969 [handler.py:handle_request():146] handle_request: run_start +2024-05-08 19:52:58,208 DEBUG HandlerThread:12969 [system_info.py:__init__():26] System info init +2024-05-08 19:52:58,209 DEBUG HandlerThread:12969 [system_info.py:__init__():41] System info init done +2024-05-08 19:52:58,209 INFO HandlerThread:12969 [system_monitor.py:start():194] Starting system monitor +2024-05-08 19:52:58,209 INFO SystemMonitor:12969 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-08 19:52:58,209 INFO HandlerThread:12969 [system_monitor.py:probe():214] Collecting system info +2024-05-08 19:52:58,210 INFO SystemMonitor:12969 [interfaces.py:start():190] Started cpu monitoring +2024-05-08 19:52:58,212 INFO SystemMonitor:12969 [interfaces.py:start():190] Started disk monitoring +2024-05-08 19:52:58,213 INFO SystemMonitor:12969 [interfaces.py:start():190] Started gpu monitoring +2024-05-08 19:52:58,213 INFO SystemMonitor:12969 [interfaces.py:start():190] Started memory monitoring +2024-05-08 19:52:58,215 INFO SystemMonitor:12969 [interfaces.py:start():190] Started network monitoring +2024-05-08 19:52:58,246 DEBUG HandlerThread:12969 [system_info.py:probe():150] Probing system +2024-05-08 19:52:58,250 DEBUG HandlerThread:12969 [system_info.py:_probe_git():135] Probing git +2024-05-08 19:52:58,262 DEBUG HandlerThread:12969 [system_info.py:_probe_git():143] Probing git done +2024-05-08 19:52:58,262 DEBUG HandlerThread:12969 [system_info.py:probe():198] Probing system done +2024-05-08 19:52:58,262 DEBUG HandlerThread:12969 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T02:52:58.246583', 'startedAt': '2024-05-09T02:52:58.086152', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.56875, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.321647644042969}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-08 19:52:58,262 INFO HandlerThread:12969 [system_monitor.py:probe():224] Finished collecting system info +2024-05-08 19:52:58,262 INFO HandlerThread:12969 [system_monitor.py:probe():227] Publishing system info +2024-05-08 19:52:58,262 DEBUG HandlerThread:12969 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-08 19:53:06,223 INFO memory :12969 [interfaces.py:monitor():140] Process proc.memory.rssMB has exited. +2024-05-08 19:53:06,223 DEBUG SystemMonitor:12969 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-08 19:53:06,225 DEBUG SystemMonitor:12969 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-08 19:53:06,226 DEBUG SystemMonitor:12969 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-08 19:53:06,394 DEBUG HandlerThread:12969 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-08 19:53:06,400 INFO HandlerThread:12969 [system_monitor.py:probe():229] Finished publishing system info +2024-05-08 19:53:06,406 DEBUG HandlerThread:12969 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:53:06,406 DEBUG HandlerThread:12969 [handler.py:handle_request():146] handle_request: keepalive +2024-05-08 19:53:06,407 DEBUG SenderThread:12969 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:53:07,114 WARNING StreamThr :12969 [internal.py:is_dead():414] Internal process exiting, parent pid 12756 disappeared +2024-05-08 19:53:07,114 ERROR StreamThr :12969 [internal.py:wandb_internal():152] Internal process shutdown. +2024-05-08 19:53:07,406 INFO HandlerThread:12969 [handler.py:finish():866] shutting down handler +2024-05-08 19:53:07,406 INFO HandlerThread:12969 [system_monitor.py:finish():203] Stopping system monitor +2024-05-08 19:53:07,406 INFO HandlerThread:12969 [interfaces.py:finish():202] Joined cpu monitor +2024-05-08 19:53:07,407 INFO HandlerThread:12969 [interfaces.py:finish():202] Joined disk monitor +2024-05-08 19:53:07,407 INFO WriterThread:12969 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_195258-zu5yru3s/run-zu5yru3s.wandb +2024-05-08 19:53:07,407 INFO HandlerThread:12969 [interfaces.py:finish():202] Joined gpu monitor +2024-05-08 19:53:07,407 INFO SenderThread:12969 [sender.py:finish():1546] shutting down sender +2024-05-08 19:53:07,408 INFO HandlerThread:12969 [interfaces.py:finish():202] Joined memory monitor +2024-05-08 19:53:07,408 INFO HandlerThread:12969 [interfaces.py:finish():202] Joined network monitor diff --git a/wandb/offline-run-20240508_195258-zu5yru3s/logs/debug.log b/wandb/offline-run-20240508_195258-zu5yru3s/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..2f8681163d7d8268df74a62773db8788a82a4443 --- /dev/null +++ b/wandb/offline-run-20240508_195258-zu5yru3s/logs/debug.log @@ -0,0 +1,21 @@ +2024-05-08 19:52:58,097 INFO MainThread:12756 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-08 19:52:58,098 INFO MainThread:12756 [wandb_setup.py:_flush():76] Configure stats pid to 12756 +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_195258-zu5yru3s/logs/debug.log +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_195258-zu5yru3s/logs/debug-internal.log +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_init.py:init():561] calling init triggers +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_init.py:init():611] starting backend +2024-05-08 19:52:58,099 INFO MainThread:12756 [wandb_init.py:init():615] setting up manager +2024-05-08 19:52:58,106 INFO MainThread:12756 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-08 19:52:58,112 INFO MainThread:12756 [wandb_init.py:init():623] backend started and connected +2024-05-08 19:52:58,122 INFO MainThread:12756 [wandb_init.py:init():715] updated telemetry +2024-05-08 19:52:58,132 INFO MainThread:12756 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-08 19:52:58,140 INFO MainThread:12756 [wandb_init.py:init():799] starting run threads in backend diff --git a/wandb/offline-run-20240508_195258-zu5yru3s/run-zu5yru3s.wandb b/wandb/offline-run-20240508_195258-zu5yru3s/run-zu5yru3s.wandb new file mode 100644 index 0000000000000000000000000000000000000000..19a4112f9f4a1659633cb547cfe0373b255d5453 Binary files /dev/null and b/wandb/offline-run-20240508_195258-zu5yru3s/run-zu5yru3s.wandb differ diff --git a/wandb/offline-run-20240508_195517-9qhnqfwd/files/conda-environment.yaml b/wandb/offline-run-20240508_195517-9qhnqfwd/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ed56ab5ac90a4bed6f79ed5c79ec57e153ec12f --- /dev/null +++ b/wandb/offline-run-20240508_195517-9qhnqfwd/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.37.2 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240508_195517-9qhnqfwd/files/wandb-metadata.json b/wandb/offline-run-20240508_195517-9qhnqfwd/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..fe89cb8d6b12e925a0b46341e6afbc44a5833ebb --- /dev/null +++ b/wandb/offline-run-20240508_195517-9qhnqfwd/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T02:55:17.768074", + "startedAt": "2024-05-09T02:55:17.606782", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.6, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.321647644042969 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240508_195517-9qhnqfwd/logs/debug-internal.log b/wandb/offline-run-20240508_195517-9qhnqfwd/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..ae0e1bb068b87c26f5f8ead9d75659199036348b --- /dev/null +++ b/wandb/offline-run-20240508_195517-9qhnqfwd/logs/debug-internal.log @@ -0,0 +1,42 @@ +2024-05-08 19:55:17,632 INFO StreamThr :13683 [internal.py:wandb_internal():86] W&B internal server running at pid: 13683, started at: 2024-05-08 19:55:17.630812 +2024-05-08 19:55:17,635 DEBUG HandlerThread:13683 [handler.py:handle_request():146] handle_request: status +2024-05-08 19:55:17,641 INFO WriterThread:13683 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_195517-9qhnqfwd/run-9qhnqfwd.wandb +2024-05-08 19:55:17,706 DEBUG HandlerThread:13683 [handler.py:handle_request():146] handle_request: run_start +2024-05-08 19:55:17,733 DEBUG HandlerThread:13683 [system_info.py:__init__():26] System info init +2024-05-08 19:55:17,733 DEBUG HandlerThread:13683 [system_info.py:__init__():41] System info init done +2024-05-08 19:55:17,733 INFO HandlerThread:13683 [system_monitor.py:start():194] Starting system monitor +2024-05-08 19:55:17,734 INFO SystemMonitor:13683 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-08 19:55:17,734 INFO HandlerThread:13683 [system_monitor.py:probe():214] Collecting system info +2024-05-08 19:55:17,735 INFO SystemMonitor:13683 [interfaces.py:start():190] Started cpu monitoring +2024-05-08 19:55:17,736 INFO SystemMonitor:13683 [interfaces.py:start():190] Started disk monitoring +2024-05-08 19:55:17,738 INFO SystemMonitor:13683 [interfaces.py:start():190] Started gpu monitoring +2024-05-08 19:55:17,739 INFO SystemMonitor:13683 [interfaces.py:start():190] Started memory monitoring +2024-05-08 19:55:17,740 INFO SystemMonitor:13683 [interfaces.py:start():190] Started network monitoring +2024-05-08 19:55:17,767 DEBUG HandlerThread:13683 [system_info.py:probe():150] Probing system +2024-05-08 19:55:17,770 DEBUG HandlerThread:13683 [system_info.py:_probe_git():135] Probing git +2024-05-08 19:55:17,782 DEBUG HandlerThread:13683 [system_info.py:_probe_git():143] Probing git done +2024-05-08 19:55:17,782 DEBUG HandlerThread:13683 [system_info.py:probe():198] Probing system done +2024-05-08 19:55:17,782 DEBUG HandlerThread:13683 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T02:55:17.768074', 'startedAt': '2024-05-09T02:55:17.606782', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.6, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.321647644042969}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-08 19:55:17,782 INFO HandlerThread:13683 [system_monitor.py:probe():224] Finished collecting system info +2024-05-08 19:55:17,782 INFO HandlerThread:13683 [system_monitor.py:probe():227] Publishing system info +2024-05-08 19:55:17,782 DEBUG HandlerThread:13683 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-08 19:55:25,715 DEBUG HandlerThread:13683 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-08 19:55:25,719 INFO HandlerThread:13683 [system_monitor.py:probe():229] Finished publishing system info +2024-05-08 19:55:25,724 DEBUG HandlerThread:13683 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:55:25,724 DEBUG HandlerThread:13683 [handler.py:handle_request():146] handle_request: keepalive +2024-05-08 19:55:25,725 DEBUG SenderThread:13683 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:55:26,035 DEBUG HandlerThread:13683 [handler.py:handle_request():146] handle_request: python_packages +2024-05-08 19:55:27,725 DEBUG HandlerThread:13683 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:55:27,726 DEBUG SenderThread:13683 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:55:32,727 DEBUG HandlerThread:13683 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:55:32,727 DEBUG SenderThread:13683 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:55:37,728 DEBUG HandlerThread:13683 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 19:55:37,728 DEBUG SenderThread:13683 [sender.py:send_request():406] send_request: status_report +2024-05-08 19:55:41,764 INFO cpu :13683 [interfaces.py:monitor():140] Process cpu has exited. +2024-05-08 19:55:41,765 DEBUG SystemMonitor:13683 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-08 19:55:41,766 DEBUG SystemMonitor:13683 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-08 19:55:41,766 DEBUG SystemMonitor:13683 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-08 19:55:41,770 INFO memory :13683 [interfaces.py:monitor():140] Process proc.memory.rssMB has exited. +2024-05-08 19:55:42,638 WARNING StreamThr :13683 [internal.py:is_dead():414] Internal process exiting, parent pid 13472 disappeared +2024-05-08 19:55:42,638 ERROR StreamThr :13683 [internal.py:wandb_internal():152] Internal process shutdown. +2024-05-08 19:55:42,672 INFO MainThread:13683 [internal.py:handle_exit():76] Internal process exited diff --git a/wandb/offline-run-20240508_195517-9qhnqfwd/logs/debug.log b/wandb/offline-run-20240508_195517-9qhnqfwd/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..03012baec4a193a0bb0053213d5f372cabee108a --- /dev/null +++ b/wandb/offline-run-20240508_195517-9qhnqfwd/logs/debug.log @@ -0,0 +1,26 @@ +2024-05-08 19:55:17,620 INFO MainThread:13472 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_setup.py:_flush():76] Configure stats pid to 13472 +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_195517-9qhnqfwd/logs/debug.log +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_195517-9qhnqfwd/logs/debug-internal.log +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_init.py:init():561] calling init triggers +2024-05-08 19:55:17,622 INFO MainThread:13472 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-08 19:55:17,623 INFO MainThread:13472 [wandb_init.py:init():611] starting backend +2024-05-08 19:55:17,623 INFO MainThread:13472 [wandb_init.py:init():615] setting up manager +2024-05-08 19:55:17,628 INFO MainThread:13472 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-08 19:55:17,631 INFO MainThread:13472 [wandb_init.py:init():623] backend started and connected +2024-05-08 19:55:17,641 INFO MainThread:13472 [wandb_init.py:init():715] updated telemetry +2024-05-08 19:55:17,653 INFO MainThread:13472 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-08 19:55:17,660 INFO MainThread:13472 [wandb_init.py:init():799] starting run threads in backend +2024-05-08 19:55:26,035 INFO MainThread:13472 [wandb_run.py:_console_start():2335] atexit reg +2024-05-08 19:55:26,035 INFO MainThread:13472 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-08 19:55:26,035 INFO MainThread:13472 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-08 19:55:26,035 INFO MainThread:13472 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-08 19:55:26,036 INFO MainThread:13472 [wandb_init.py:init():842] run started, returning control to user process diff --git a/wandb/offline-run-20240508_195517-9qhnqfwd/run-9qhnqfwd.wandb b/wandb/offline-run-20240508_195517-9qhnqfwd/run-9qhnqfwd.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/wandb/offline-run-20240508_200327-7vse962c/files/conda-environment.yaml b/wandb/offline-run-20240508_200327-7vse962c/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb5c46cd8a524560371e6601fcdbf9ef63efe33a --- /dev/null +++ b/wandb/offline-run-20240508_200327-7vse962c/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.40.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240508_200327-7vse962c/files/wandb-metadata.json b/wandb/offline-run-20240508_200327-7vse962c/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..57607e0dd519f96d1b73a18c227dc8301b7953dd --- /dev/null +++ b/wandb/offline-run-20240508_200327-7vse962c/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T03:03:28.093390", + "startedAt": "2024-05-09T03:03:27.902237", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.5734375, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.2, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.323635101318359 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240508_200327-7vse962c/files/wandb-summary.json b/wandb/offline-run-20240508_200327-7vse962c/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..30fe69f73261d9403d3ab1103903e365d1dc3a2f --- /dev/null +++ b/wandb/offline-run-20240508_200327-7vse962c/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 20}} \ No newline at end of file diff --git a/wandb/offline-run-20240508_200327-7vse962c/logs/debug-internal.log b/wandb/offline-run-20240508_200327-7vse962c/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4e4b33bd912755f4f924fef6497c587a0396cdb8 --- /dev/null +++ b/wandb/offline-run-20240508_200327-7vse962c/logs/debug-internal.log @@ -0,0 +1,140 @@ +2024-05-08 20:03:27,941 INFO StreamThr :14700 [internal.py:wandb_internal():86] W&B internal server running at pid: 14700, started at: 2024-05-08 20:03:27.939039 +2024-05-08 20:03:27,944 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: status +2024-05-08 20:03:27,950 INFO WriterThread:14700 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_200327-7vse962c/run-7vse962c.wandb +2024-05-08 20:03:28,024 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: run_start +2024-05-08 20:03:28,046 DEBUG HandlerThread:14700 [system_info.py:__init__():26] System info init +2024-05-08 20:03:28,046 DEBUG HandlerThread:14700 [system_info.py:__init__():41] System info init done +2024-05-08 20:03:28,046 INFO HandlerThread:14700 [system_monitor.py:start():194] Starting system monitor +2024-05-08 20:03:28,046 INFO SystemMonitor:14700 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-08 20:03:28,046 INFO HandlerThread:14700 [system_monitor.py:probe():214] Collecting system info +2024-05-08 20:03:28,057 INFO SystemMonitor:14700 [interfaces.py:start():190] Started cpu monitoring +2024-05-08 20:03:28,060 INFO SystemMonitor:14700 [interfaces.py:start():190] Started disk monitoring +2024-05-08 20:03:28,064 INFO SystemMonitor:14700 [interfaces.py:start():190] Started gpu monitoring +2024-05-08 20:03:28,064 INFO SystemMonitor:14700 [interfaces.py:start():190] Started memory monitoring +2024-05-08 20:03:28,065 INFO SystemMonitor:14700 [interfaces.py:start():190] Started network monitoring +2024-05-08 20:03:28,093 DEBUG HandlerThread:14700 [system_info.py:probe():150] Probing system +2024-05-08 20:03:28,097 DEBUG HandlerThread:14700 [system_info.py:_probe_git():135] Probing git +2024-05-08 20:03:28,112 DEBUG HandlerThread:14700 [system_info.py:_probe_git():143] Probing git done +2024-05-08 20:03:28,113 DEBUG HandlerThread:14700 [system_info.py:probe():198] Probing system done +2024-05-08 20:03:28,113 DEBUG HandlerThread:14700 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T03:03:28.093390', 'startedAt': '2024-05-09T03:03:27.902237', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.5734375, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.2, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.323635101318359}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-08 20:03:28,113 INFO HandlerThread:14700 [system_monitor.py:probe():224] Finished collecting system info +2024-05-08 20:03:28,113 INFO HandlerThread:14700 [system_monitor.py:probe():227] Publishing system info +2024-05-08 20:03:28,113 DEBUG HandlerThread:14700 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-08 20:03:36,245 DEBUG HandlerThread:14700 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-08 20:03:36,250 INFO HandlerThread:14700 [system_monitor.py:probe():229] Finished publishing system info +2024-05-08 20:03:36,255 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 20:03:36,255 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: keepalive +2024-05-08 20:03:36,255 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: status_report +2024-05-08 20:03:36,520 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: python_packages +2024-05-08 20:03:38,256 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 20:03:38,256 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: status_report +2024-05-08 20:03:43,257 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 20:03:43,257 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: status_report +2024-05-08 20:03:48,258 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 20:03:48,258 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: status_report +2024-05-08 20:03:48,995 DEBUG SenderThread:14700 [sender.py:send():379] send: exit +2024-05-08 20:03:48,995 INFO SenderThread:14700 [sender.py:send_exit():586] handling exit code: 1 +2024-05-08 20:03:48,995 INFO SenderThread:14700 [sender.py:send_exit():588] handling runtime: 20 +2024-05-08 20:03:48,997 INFO SenderThread:14700 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-08 20:03:48,997 INFO SenderThread:14700 [sender.py:send_exit():594] send defer +2024-05-08 20:03:48,998 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:48,998 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 0 +2024-05-08 20:03:48,998 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:48,998 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-05-08 20:03:48,998 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 1 +2024-05-08 20:03:48,998 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:48,998 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 1 +2024-05-08 20:03:48,998 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:48,998 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-05-08 20:03:48,999 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 2 +2024-05-08 20:03:48,999 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:48,999 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 2 +2024-05-08 20:03:48,999 INFO HandlerThread:14700 [system_monitor.py:finish():203] Stopping system monitor +2024-05-08 20:03:48,999 DEBUG SystemMonitor:14700 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-08 20:03:49,000 DEBUG SystemMonitor:14700 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-08 20:03:49,000 INFO HandlerThread:14700 [interfaces.py:finish():202] Joined cpu monitor +2024-05-08 20:03:49,000 DEBUG SystemMonitor:14700 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-08 20:03:49,000 INFO HandlerThread:14700 [interfaces.py:finish():202] Joined disk monitor +2024-05-08 20:03:49,043 INFO HandlerThread:14700 [interfaces.py:finish():202] Joined gpu monitor +2024-05-08 20:03:49,043 INFO HandlerThread:14700 [interfaces.py:finish():202] Joined memory monitor +2024-05-08 20:03:49,043 INFO HandlerThread:14700 [interfaces.py:finish():202] Joined network monitor +2024-05-08 20:03:49,044 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,044 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-05-08 20:03:49,044 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 3 +2024-05-08 20:03:49,044 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,044 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 3 +2024-05-08 20:03:49,044 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,045 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-05-08 20:03:49,045 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 4 +2024-05-08 20:03:49,045 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,045 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 4 +2024-05-08 20:03:49,045 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,045 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-05-08 20:03:49,045 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 5 +2024-05-08 20:03:49,045 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,046 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 5 +2024-05-08 20:03:49,046 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,046 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-05-08 20:03:49,046 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 6 +2024-05-08 20:03:49,046 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,046 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 6 +2024-05-08 20:03:49,046 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,046 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-05-08 20:03:49,047 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 7 +2024-05-08 20:03:49,047 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 20:03:49,047 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,047 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 7 +2024-05-08 20:03:49,047 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: status_report +2024-05-08 20:03:49,047 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,047 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-05-08 20:03:49,047 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 8 +2024-05-08 20:03:49,048 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,048 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 8 +2024-05-08 20:03:49,048 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,048 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-05-08 20:03:49,048 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 9 +2024-05-08 20:03:49,048 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,048 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 9 +2024-05-08 20:03:49,048 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,048 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-05-08 20:03:49,048 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 10 +2024-05-08 20:03:49,048 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,049 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 10 +2024-05-08 20:03:49,049 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,049 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-05-08 20:03:49,049 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 11 +2024-05-08 20:03:49,049 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,049 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 11 +2024-05-08 20:03:49,049 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,049 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-05-08 20:03:49,049 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 12 +2024-05-08 20:03:49,049 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,049 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 12 +2024-05-08 20:03:49,050 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,050 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-05-08 20:03:49,050 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 13 +2024-05-08 20:03:49,050 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,050 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 13 +2024-05-08 20:03:49,050 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,050 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-05-08 20:03:49,050 INFO SenderThread:14700 [sender.py:transition_state():614] send defer: 14 +2024-05-08 20:03:49,050 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: defer +2024-05-08 20:03:49,051 DEBUG SenderThread:14700 [sender.py:send():379] send: final +2024-05-08 20:03:49,051 INFO HandlerThread:14700 [handler.py:handle_request_defer():172] handle defer: 14 +2024-05-08 20:03:49,051 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: defer +2024-05-08 20:03:49,051 INFO SenderThread:14700 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-05-08 20:03:49,052 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-08 20:03:49,052 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: poll_exit +2024-05-08 20:03:49,052 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-08 20:03:49,052 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: poll_exit +2024-05-08 20:03:49,053 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: server_info +2024-05-08 20:03:49,053 DEBUG SenderThread:14700 [sender.py:send_request():406] send_request: server_info +2024-05-08 20:03:49,053 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: get_summary +2024-05-08 20:03:49,054 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: sampled_history +2024-05-08 20:03:49,054 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-08 20:03:49,054 INFO MainThread:14700 [wandb_run.py:_footer_history_summary_info():3936] rendering history +2024-05-08 20:03:49,054 INFO MainThread:14700 [wandb_run.py:_footer_history_summary_info():3968] rendering summary +2024-05-08 20:03:49,055 DEBUG HandlerThread:14700 [handler.py:handle_request():146] handle_request: shutdown +2024-05-08 20:03:49,055 INFO HandlerThread:14700 [handler.py:finish():866] shutting down handler +2024-05-08 20:03:50,053 INFO WriterThread:14700 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_200327-7vse962c/run-7vse962c.wandb +2024-05-08 20:03:50,053 INFO SenderThread:14700 [sender.py:finish():1546] shutting down sender diff --git a/wandb/offline-run-20240508_200327-7vse962c/logs/debug.log b/wandb/offline-run-20240508_200327-7vse962c/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..b7de248073acbc7eac752e0aded60bbbba0fdb42 --- /dev/null +++ b/wandb/offline-run-20240508_200327-7vse962c/logs/debug.log @@ -0,0 +1,27 @@ +2024-05-08 20:03:27,917 INFO MainThread:14450 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-08 20:03:27,918 INFO MainThread:14450 [wandb_setup.py:_flush():76] Configure stats pid to 14450 +2024-05-08 20:03:27,918 INFO MainThread:14450 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_200327-7vse962c/logs/debug.log +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_200327-7vse962c/logs/debug-internal.log +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_init.py:init():561] calling init triggers +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_init.py:init():611] starting backend +2024-05-08 20:03:27,919 INFO MainThread:14450 [wandb_init.py:init():615] setting up manager +2024-05-08 20:03:27,937 INFO MainThread:14450 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-08 20:03:27,939 INFO MainThread:14450 [wandb_init.py:init():623] backend started and connected +2024-05-08 20:03:27,949 INFO MainThread:14450 [wandb_init.py:init():715] updated telemetry +2024-05-08 20:03:27,970 INFO MainThread:14450 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-08 20:03:27,978 INFO MainThread:14450 [wandb_init.py:init():799] starting run threads in backend +2024-05-08 20:03:36,520 INFO MainThread:14450 [wandb_run.py:_console_start():2335] atexit reg +2024-05-08 20:03:36,520 INFO MainThread:14450 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-08 20:03:36,520 INFO MainThread:14450 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-08 20:03:36,520 INFO MainThread:14450 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-08 20:03:36,521 INFO MainThread:14450 [wandb_init.py:init():842] run started, returning control to user process +2024-05-08 20:03:50,056 WARNING MsgRouterThr:14450 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/offline-run-20240508_200327-7vse962c/run-7vse962c.wandb b/wandb/offline-run-20240508_200327-7vse962c/run-7vse962c.wandb new file mode 100644 index 0000000000000000000000000000000000000000..6c16fa013d6e840633c353a3f5a900448f9b022a Binary files /dev/null and b/wandb/offline-run-20240508_200327-7vse962c/run-7vse962c.wandb differ diff --git a/wandb/offline-run-20240508_200654-93gxjtkh/files/conda-environment.yaml b/wandb/offline-run-20240508_200654-93gxjtkh/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb5c46cd8a524560371e6601fcdbf9ef63efe33a --- /dev/null +++ b/wandb/offline-run-20240508_200654-93gxjtkh/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.40.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240508_200654-93gxjtkh/files/wandb-metadata.json b/wandb/offline-run-20240508_200654-93gxjtkh/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..7d0c37cf93ac8eeeb5a3cab148342264af3f16aa --- /dev/null +++ b/wandb/offline-run-20240508_200654-93gxjtkh/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T03:06:54.645991", + "startedAt": "2024-05-09T03:06:54.494591", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-16n02.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.6, + "min": 1500.0, + "max": 2500.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.2, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 2.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2500.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.321681976318359 + } + }, + "gpu": "NVIDIA A100-SXM4-40GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + }, + { + "name": "NVIDIA A100-SXM4-40GB", + "memory_total": 42949672960 + } + ], + "memory": { + "total": 503.587890625 + } +} diff --git a/wandb/offline-run-20240508_200654-93gxjtkh/logs/debug-internal.log b/wandb/offline-run-20240508_200654-93gxjtkh/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..db9bcc70f1f82a704bdde2221d15e27e252224a7 --- /dev/null +++ b/wandb/offline-run-20240508_200654-93gxjtkh/logs/debug-internal.log @@ -0,0 +1,48 @@ +2024-05-08 20:06:54,517 INFO StreamThr :15787 [internal.py:wandb_internal():86] W&B internal server running at pid: 15787, started at: 2024-05-08 20:06:54.515352 +2024-05-08 20:06:54,520 DEBUG HandlerThread:15787 [handler.py:handle_request():146] handle_request: status +2024-05-08 20:06:54,526 INFO WriterThread:15787 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_200654-93gxjtkh/run-93gxjtkh.wandb +2024-05-08 20:06:54,586 DEBUG HandlerThread:15787 [handler.py:handle_request():146] handle_request: run_start +2024-05-08 20:06:54,608 DEBUG HandlerThread:15787 [system_info.py:__init__():26] System info init +2024-05-08 20:06:54,608 DEBUG HandlerThread:15787 [system_info.py:__init__():41] System info init done +2024-05-08 20:06:54,608 INFO HandlerThread:15787 [system_monitor.py:start():194] Starting system monitor +2024-05-08 20:06:54,608 INFO SystemMonitor:15787 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-08 20:06:54,608 INFO HandlerThread:15787 [system_monitor.py:probe():214] Collecting system info +2024-05-08 20:06:54,609 INFO SystemMonitor:15787 [interfaces.py:start():190] Started cpu monitoring +2024-05-08 20:06:54,611 INFO SystemMonitor:15787 [interfaces.py:start():190] Started disk monitoring +2024-05-08 20:06:54,613 INFO SystemMonitor:15787 [interfaces.py:start():190] Started gpu monitoring +2024-05-08 20:06:54,614 INFO SystemMonitor:15787 [interfaces.py:start():190] Started memory monitoring +2024-05-08 20:06:54,615 INFO SystemMonitor:15787 [interfaces.py:start():190] Started network monitoring +2024-05-08 20:06:54,645 DEBUG HandlerThread:15787 [system_info.py:probe():150] Probing system +2024-05-08 20:06:54,649 DEBUG HandlerThread:15787 [system_info.py:_probe_git():135] Probing git +2024-05-08 20:06:54,662 DEBUG HandlerThread:15787 [system_info.py:_probe_git():143] Probing git done +2024-05-08 20:06:54,662 DEBUG HandlerThread:15787 [system_info.py:probe():198] Probing system done +2024-05-08 20:06:54,662 DEBUG HandlerThread:15787 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.118.1.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T03:06:54.645991', 'startedAt': '2024-05-09T03:06:54.494591', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-16n02.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.6, 'min': 1500.0, 'max': 2500.0}, 'cpu_freq_per_core': [{'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.2, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 2.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}, {'current': 1.5, 'min': 1500.0, 'max': 2500.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.321681976318359}}, 'gpu': 'NVIDIA A100-SXM4-40GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}, {'name': 'NVIDIA A100-SXM4-40GB', 'memory_total': 42949672960}], 'memory': {'total': 503.587890625}} +2024-05-08 20:06:54,662 INFO HandlerThread:15787 [system_monitor.py:probe():224] Finished collecting system info +2024-05-08 20:06:54,662 INFO HandlerThread:15787 [system_monitor.py:probe():227] Publishing system info +2024-05-08 20:06:54,662 DEBUG HandlerThread:15787 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-08 20:07:02,745 DEBUG HandlerThread:15787 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-08 20:07:02,748 INFO HandlerThread:15787 [system_monitor.py:probe():229] Finished publishing system info +2024-05-08 20:07:02,753 DEBUG HandlerThread:15787 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 20:07:02,753 DEBUG HandlerThread:15787 [handler.py:handle_request():146] handle_request: keepalive +2024-05-08 20:07:02,753 DEBUG SenderThread:15787 [sender.py:send_request():406] send_request: status_report +2024-05-08 20:07:03,016 DEBUG HandlerThread:15787 [handler.py:handle_request():146] handle_request: python_packages +2024-05-08 20:07:04,754 DEBUG HandlerThread:15787 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 20:07:04,755 DEBUG SenderThread:15787 [sender.py:send_request():406] send_request: status_report +2024-05-08 20:07:09,756 DEBUG HandlerThread:15787 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 20:07:09,756 DEBUG SenderThread:15787 [sender.py:send_request():406] send_request: status_report +2024-05-08 20:07:14,757 DEBUG HandlerThread:15787 [handler.py:handle_request():146] handle_request: status_report +2024-05-08 20:07:14,758 DEBUG SenderThread:15787 [sender.py:send_request():406] send_request: status_report +2024-05-08 20:07:19,525 WARNING StreamThr :15787 [internal.py:is_dead():414] Internal process exiting, parent pid 15577 disappeared +2024-05-08 20:07:19,525 ERROR StreamThr :15787 [internal.py:wandb_internal():152] Internal process shutdown. +2024-05-08 20:07:19,758 INFO HandlerThread:15787 [handler.py:finish():866] shutting down handler +2024-05-08 20:07:19,758 INFO WriterThread:15787 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_200654-93gxjtkh/run-93gxjtkh.wandb +2024-05-08 20:07:19,758 INFO HandlerThread:15787 [system_monitor.py:finish():203] Stopping system monitor +2024-05-08 20:07:19,759 INFO SenderThread:15787 [sender.py:finish():1546] shutting down sender +2024-05-08 20:07:19,759 DEBUG SystemMonitor:15787 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-08 20:07:19,760 INFO HandlerThread:15787 [interfaces.py:finish():202] Joined cpu monitor +2024-05-08 20:07:19,761 DEBUG SystemMonitor:15787 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-08 20:07:19,762 INFO HandlerThread:15787 [interfaces.py:finish():202] Joined disk monitor +2024-05-08 20:07:19,762 DEBUG SystemMonitor:15787 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-08 20:07:19,763 INFO HandlerThread:15787 [interfaces.py:finish():202] Joined gpu monitor +2024-05-08 20:07:19,765 INFO HandlerThread:15787 [interfaces.py:finish():202] Joined memory monitor +2024-05-08 20:07:19,765 INFO HandlerThread:15787 [interfaces.py:finish():202] Joined network monitor diff --git a/wandb/offline-run-20240508_200654-93gxjtkh/logs/debug.log b/wandb/offline-run-20240508_200654-93gxjtkh/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..7725981c7bb1509bef99682d1e5782b30fce0bbd --- /dev/null +++ b/wandb/offline-run-20240508_200654-93gxjtkh/logs/debug.log @@ -0,0 +1,26 @@ +2024-05-08 20:06:54,505 INFO MainThread:15577 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-08 20:06:54,506 INFO MainThread:15577 [wandb_setup.py:_flush():76] Configure stats pid to 15577 +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_200654-93gxjtkh/logs/debug.log +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240508_200654-93gxjtkh/logs/debug-internal.log +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_init.py:init():561] calling init triggers +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_init.py:init():611] starting backend +2024-05-08 20:06:54,507 INFO MainThread:15577 [wandb_init.py:init():615] setting up manager +2024-05-08 20:06:54,513 INFO MainThread:15577 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-08 20:06:54,516 INFO MainThread:15577 [wandb_init.py:init():623] backend started and connected +2024-05-08 20:06:54,525 INFO MainThread:15577 [wandb_init.py:init():715] updated telemetry +2024-05-08 20:06:54,535 INFO MainThread:15577 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-08 20:06:54,542 INFO MainThread:15577 [wandb_init.py:init():799] starting run threads in backend +2024-05-08 20:07:03,015 INFO MainThread:15577 [wandb_run.py:_console_start():2335] atexit reg +2024-05-08 20:07:03,016 INFO MainThread:15577 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-08 20:07:03,016 INFO MainThread:15577 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-08 20:07:03,016 INFO MainThread:15577 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-08 20:07:03,017 INFO MainThread:15577 [wandb_init.py:init():842] run started, returning control to user process diff --git a/wandb/offline-run-20240508_200654-93gxjtkh/run-93gxjtkh.wandb b/wandb/offline-run-20240508_200654-93gxjtkh/run-93gxjtkh.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9993b5e40afdc600d717fae92068fd32c82cefcb Binary files /dev/null and b/wandb/offline-run-20240508_200654-93gxjtkh/run-93gxjtkh.wandb differ diff --git a/wandb/offline-run-20240509_105304-nvk0evt8/files/conda-environment.yaml b/wandb/offline-run-20240509_105304-nvk0evt8/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5ab80d91aabc2199c787d0c4d29ff1e49b7f1807 --- /dev/null +++ b/wandb/offline-run-20240509_105304-nvk0evt8/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.31.0 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240509_105304-nvk0evt8/files/wandb-metadata.json b/wandb/offline-run-20240509_105304-nvk0evt8/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..af13cdeb39cb28087c66f289c4f54f305c9e9fbc --- /dev/null +++ b/wandb/offline-run-20240509_105304-nvk0evt8/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T17:53:04.631669", + "startedAt": "2024-05-09T17:53:04.465066", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "64", + "--model.finetune_per_device_batch_size", + "4", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-11n01.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.5796875, + "min": 1500.0, + "max": 2800.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.625911712646484 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5838279724121 + } +} diff --git a/wandb/offline-run-20240509_105304-nvk0evt8/logs/debug-internal.log b/wandb/offline-run-20240509_105304-nvk0evt8/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..aa2cd21d505ba87024756110f59a0ae038e9c568 --- /dev/null +++ b/wandb/offline-run-20240509_105304-nvk0evt8/logs/debug-internal.log @@ -0,0 +1,211 @@ +2024-05-09 10:53:04,490 INFO StreamThr :29544 [internal.py:wandb_internal():86] W&B internal server running at pid: 29544, started at: 2024-05-09 10:53:04.487722 +2024-05-09 10:53:04,495 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status +2024-05-09 10:53:04,503 INFO WriterThread:29544 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_105304-nvk0evt8/run-nvk0evt8.wandb +2024-05-09 10:53:04,580 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: run_start +2024-05-09 10:53:04,598 DEBUG HandlerThread:29544 [system_info.py:__init__():26] System info init +2024-05-09 10:53:04,598 DEBUG HandlerThread:29544 [system_info.py:__init__():41] System info init done +2024-05-09 10:53:04,598 INFO HandlerThread:29544 [system_monitor.py:start():194] Starting system monitor +2024-05-09 10:53:04,598 INFO SystemMonitor:29544 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-09 10:53:04,598 INFO HandlerThread:29544 [system_monitor.py:probe():214] Collecting system info +2024-05-09 10:53:04,599 INFO SystemMonitor:29544 [interfaces.py:start():190] Started cpu monitoring +2024-05-09 10:53:04,600 INFO SystemMonitor:29544 [interfaces.py:start():190] Started disk monitoring +2024-05-09 10:53:04,601 INFO SystemMonitor:29544 [interfaces.py:start():190] Started gpu monitoring +2024-05-09 10:53:04,601 INFO SystemMonitor:29544 [interfaces.py:start():190] Started memory monitoring +2024-05-09 10:53:04,602 INFO SystemMonitor:29544 [interfaces.py:start():190] Started network monitoring +2024-05-09 10:53:04,631 DEBUG HandlerThread:29544 [system_info.py:probe():150] Probing system +2024-05-09 10:53:04,635 DEBUG HandlerThread:29544 [system_info.py:_probe_git():135] Probing git +2024-05-09 10:53:04,648 DEBUG HandlerThread:29544 [system_info.py:_probe_git():143] Probing git done +2024-05-09 10:53:04,648 DEBUG HandlerThread:29544 [system_info.py:probe():198] Probing system done +2024-05-09 10:53:04,648 DEBUG HandlerThread:29544 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T17:53:04.631669', 'startedAt': '2024-05-09T17:53:04.465066', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '64', '--model.finetune_per_device_batch_size', '4', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-11n01.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.5796875, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.625911712646484}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5838279724121}} +2024-05-09 10:53:04,648 INFO HandlerThread:29544 [system_monitor.py:probe():224] Finished collecting system info +2024-05-09 10:53:04,648 INFO HandlerThread:29544 [system_monitor.py:probe():227] Publishing system info +2024-05-09 10:53:04,648 DEBUG HandlerThread:29544 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-09 10:53:13,583 DEBUG HandlerThread:29544 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-09 10:53:13,587 INFO HandlerThread:29544 [system_monitor.py:probe():229] Finished publishing system info +2024-05-09 10:53:13,600 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:53:13,600 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: keepalive +2024-05-09 10:53:13,600 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:53:14,601 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:53:14,601 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:53:14,693 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: python_packages +2024-05-09 10:53:19,602 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:53:19,602 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:53:24,603 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:53:24,604 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:53:29,605 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:53:29,605 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:53:33,796 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:53:34,606 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:53:34,606 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:53:39,607 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:53:39,608 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:53:43,081 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:53:44,608 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:53:44,608 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:53:49,609 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:53:49,609 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:53:52,059 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:53:57,159 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:53:57,159 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:00,160 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:00,160 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:01,093 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:54:04,602 DEBUG SystemMonitor:29544 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-09 10:54:05,161 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:05,161 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:09,993 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:54:10,162 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:10,162 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:15,163 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:15,163 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:18,916 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:54:20,164 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:20,165 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:25,165 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:25,166 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:28,263 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:54:30,167 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:30,167 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:35,168 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:35,168 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:37,837 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:54:40,169 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:40,169 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:45,170 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:45,170 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:47,251 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:54:50,171 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:50,172 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:55,172 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:54:55,173 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:54:55,957 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:55:00,174 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:00,174 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:05,036 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:55:05,175 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:05,175 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:10,176 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:10,176 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:14,160 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:55:15,177 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:15,178 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:20,179 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:20,179 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:24,230 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:55:25,180 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:25,180 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:30,181 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:30,181 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:33,883 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:55:35,182 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:35,183 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:40,184 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:40,184 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:43,183 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:55:45,185 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:45,186 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:50,187 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:50,199 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:55:52,807 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:55:55,200 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:55:55,200 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:00,202 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:00,202 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:01,860 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:56:05,283 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:05,284 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:10,285 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:10,285 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:11,280 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:56:15,286 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:15,286 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:20,287 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:20,288 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:20,825 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:56:25,289 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:25,289 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:30,290 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:30,290 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:30,476 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:56:35,292 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:35,292 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:39,490 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:56:40,293 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:40,294 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:45,295 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:45,295 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:49,013 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:56:50,296 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:50,296 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:55,297 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:56:55,298 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:56:57,868 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:57:00,299 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:00,299 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:05,300 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:05,301 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:06,784 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:57:10,302 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:10,302 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:15,303 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:15,304 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:16,128 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:57:20,305 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:20,305 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:25,306 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:25,307 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:25,665 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:57:30,308 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:30,308 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:34,237 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:57:35,310 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:35,310 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:40,312 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:40,312 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:43,286 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:57:45,313 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:45,313 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:50,314 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:50,316 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:57:52,370 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:57:55,317 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:57:55,317 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:00,318 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:00,319 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:01,231 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:58:05,319 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:05,320 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:10,321 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:10,322 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:10,400 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:58:15,323 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:15,323 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:20,196 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:58:20,324 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:20,325 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:25,326 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:25,326 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:29,603 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:58:30,327 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:30,327 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:35,328 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:35,329 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:38,831 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:58:40,330 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:40,330 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:45,331 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:45,332 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:48,077 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:58:50,333 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:50,333 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:55,334 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:58:55,334 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:58:56,983 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:59:00,335 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:59:00,336 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:59:05,337 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:59:05,337 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:59:06,210 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 10:59:10,338 DEBUG HandlerThread:29544 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 10:59:10,338 DEBUG SenderThread:29544 [sender.py:send_request():406] send_request: status_report +2024-05-09 10:59:12,598 INFO MainThread:29544 [internal.py:handle_exit():76] Internal process exited diff --git a/wandb/offline-run-20240509_105304-nvk0evt8/logs/debug.log b/wandb/offline-run-20240509_105304-nvk0evt8/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..4e283f8f3bdbc9b0f9abaf20414e1980296eef90 --- /dev/null +++ b/wandb/offline-run-20240509_105304-nvk0evt8/logs/debug.log @@ -0,0 +1,26 @@ +2024-05-09 10:53:04,477 INFO MainThread:29210 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_setup.py:_flush():76] Configure stats pid to 29210 +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_105304-nvk0evt8/logs/debug.log +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_105304-nvk0evt8/logs/debug-internal.log +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_init.py:init():561] calling init triggers +2024-05-09 10:53:04,478 INFO MainThread:29210 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 64, 'finetune_per_device_batch_size': 4, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-09 10:53:04,479 INFO MainThread:29210 [wandb_init.py:init():611] starting backend +2024-05-09 10:53:04,479 INFO MainThread:29210 [wandb_init.py:init():615] setting up manager +2024-05-09 10:53:04,486 INFO MainThread:29210 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-09 10:53:04,492 INFO MainThread:29210 [wandb_init.py:init():623] backend started and connected +2024-05-09 10:53:04,503 INFO MainThread:29210 [wandb_init.py:init():715] updated telemetry +2024-05-09 10:53:04,530 INFO MainThread:29210 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-09 10:53:04,536 INFO MainThread:29210 [wandb_init.py:init():799] starting run threads in backend +2024-05-09 10:53:14,692 INFO MainThread:29210 [wandb_run.py:_console_start():2335] atexit reg +2024-05-09 10:53:14,692 INFO MainThread:29210 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-09 10:53:14,693 INFO MainThread:29210 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-09 10:53:14,693 INFO MainThread:29210 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-09 10:53:14,693 INFO MainThread:29210 [wandb_init.py:init():842] run started, returning control to user process diff --git a/wandb/offline-run-20240509_105304-nvk0evt8/run-nvk0evt8.wandb b/wandb/offline-run-20240509_105304-nvk0evt8/run-nvk0evt8.wandb new file mode 100644 index 0000000000000000000000000000000000000000..c636fde872cd3d36179c6e2d0600c1dad43df3d0 Binary files /dev/null and b/wandb/offline-run-20240509_105304-nvk0evt8/run-nvk0evt8.wandb differ diff --git a/wandb/offline-run-20240509_110904-sc0bbka3/files/conda-environment.yaml b/wandb/offline-run-20240509_110904-sc0bbka3/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb5c46cd8a524560371e6601fcdbf9ef63efe33a --- /dev/null +++ b/wandb/offline-run-20240509_110904-sc0bbka3/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.40.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240509_110904-sc0bbka3/files/wandb-metadata.json b/wandb/offline-run-20240509_110904-sc0bbka3/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..a729d2953f8efd6ac8843525e0159ba2477d987e --- /dev/null +++ b/wandb/offline-run-20240509_110904-sc0bbka3/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T18:09:04.239601", + "startedAt": "2024-05-09T18:09:04.080025", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "128", + "--model.finetune_per_device_batch_size", + "8", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-11n01.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.5890624999999998, + "min": 1500.0, + "max": 2800.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.625720977783203 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5838279724121 + } +} diff --git a/wandb/offline-run-20240509_110904-sc0bbka3/logs/debug-internal.log b/wandb/offline-run-20240509_110904-sc0bbka3/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..8f789e282e5c5e7fca07d5436dcd95326dd8d801 --- /dev/null +++ b/wandb/offline-run-20240509_110904-sc0bbka3/logs/debug-internal.log @@ -0,0 +1,499 @@ +2024-05-09 11:09:04,114 INFO StreamThr :31571 [internal.py:wandb_internal():86] W&B internal server running at pid: 31571, started at: 2024-05-09 11:09:04.111941 +2024-05-09 11:09:04,117 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status +2024-05-09 11:09:04,125 INFO WriterThread:31571 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_110904-sc0bbka3/run-sc0bbka3.wandb +2024-05-09 11:09:04,186 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: run_start +2024-05-09 11:09:04,206 DEBUG HandlerThread:31571 [system_info.py:__init__():26] System info init +2024-05-09 11:09:04,206 DEBUG HandlerThread:31571 [system_info.py:__init__():41] System info init done +2024-05-09 11:09:04,206 INFO HandlerThread:31571 [system_monitor.py:start():194] Starting system monitor +2024-05-09 11:09:04,207 INFO SystemMonitor:31571 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-09 11:09:04,207 INFO HandlerThread:31571 [system_monitor.py:probe():214] Collecting system info +2024-05-09 11:09:04,208 INFO SystemMonitor:31571 [interfaces.py:start():190] Started cpu monitoring +2024-05-09 11:09:04,209 INFO SystemMonitor:31571 [interfaces.py:start():190] Started disk monitoring +2024-05-09 11:09:04,210 INFO SystemMonitor:31571 [interfaces.py:start():190] Started gpu monitoring +2024-05-09 11:09:04,211 INFO SystemMonitor:31571 [interfaces.py:start():190] Started memory monitoring +2024-05-09 11:09:04,212 INFO SystemMonitor:31571 [interfaces.py:start():190] Started network monitoring +2024-05-09 11:09:04,239 DEBUG HandlerThread:31571 [system_info.py:probe():150] Probing system +2024-05-09 11:09:04,243 DEBUG HandlerThread:31571 [system_info.py:_probe_git():135] Probing git +2024-05-09 11:09:04,254 DEBUG HandlerThread:31571 [system_info.py:_probe_git():143] Probing git done +2024-05-09 11:09:04,254 DEBUG HandlerThread:31571 [system_info.py:probe():198] Probing system done +2024-05-09 11:09:04,254 DEBUG HandlerThread:31571 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T18:09:04.239601', 'startedAt': '2024-05-09T18:09:04.080025', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '128', '--model.finetune_per_device_batch_size', '8', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-11n01.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.5890624999999998, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.625720977783203}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5838279724121}} +2024-05-09 11:09:04,254 INFO HandlerThread:31571 [system_monitor.py:probe():224] Finished collecting system info +2024-05-09 11:09:04,254 INFO HandlerThread:31571 [system_monitor.py:probe():227] Publishing system info +2024-05-09 11:09:04,254 DEBUG HandlerThread:31571 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-09 11:09:10,984 DEBUG HandlerThread:31571 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-09 11:09:10,989 INFO HandlerThread:31571 [system_monitor.py:probe():229] Finished publishing system info +2024-05-09 11:09:10,995 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:09:10,996 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: keepalive +2024-05-09 11:09:10,996 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:09:11,216 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: python_packages +2024-05-09 11:09:14,997 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:09:14,997 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:09:19,998 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:09:19,998 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:09:24,999 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:09:24,999 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:09:30,000 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:09:30,000 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:09:33,643 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:09:35,001 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:09:35,001 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:09:40,002 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:09:40,002 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:09:45,002 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:09:45,003 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:09:50,003 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:09:50,004 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:09:52,770 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:09:55,004 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:09:55,004 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:00,005 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:00,005 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:04,212 DEBUG SystemMonitor:31571 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-09 11:10:05,006 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:05,006 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:09,797 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:10:10,007 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:10,007 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:15,008 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:15,008 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:20,009 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:20,009 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:25,010 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:25,010 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:27,312 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:10:30,011 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:30,011 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:35,011 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:35,012 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:40,012 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:40,013 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:44,442 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:10:45,013 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:45,014 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:50,014 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:50,015 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:10:55,015 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:10:55,016 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:00,016 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:00,017 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:01,144 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:11:05,017 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:05,018 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:10,019 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:10,019 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:15,020 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:15,020 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:18,961 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:11:20,021 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:20,021 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:25,022 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:25,022 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:30,023 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:30,023 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:35,024 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:35,024 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:36,526 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:11:40,025 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:40,025 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:45,026 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:45,026 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:50,026 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:50,027 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:11:53,522 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:11:55,027 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:11:55,027 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:00,028 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:00,028 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:05,029 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:05,029 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:10,030 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:10,030 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:10,451 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:12:15,031 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:15,031 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:20,031 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:20,032 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:25,032 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:25,033 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:27,346 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:12:30,033 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:30,033 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:35,034 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:35,034 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:40,035 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:40,035 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:44,166 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:12:45,036 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:45,036 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:50,036 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:50,037 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:12:55,037 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:12:55,037 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:00,038 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:00,038 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:01,660 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:13:05,039 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:05,039 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:10,040 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:10,040 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:15,041 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:15,041 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:18,952 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:13:20,041 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:20,042 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:25,042 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:25,042 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:30,043 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:30,043 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:35,044 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:35,044 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:35,878 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:13:40,045 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:40,045 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:45,045 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:45,046 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:50,046 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:50,047 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:13:53,395 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:13:55,047 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:13:55,048 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:00,048 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:00,048 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:05,049 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:05,049 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:10,050 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:10,050 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:11,210 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:14:15,051 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:15,051 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:20,052 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:20,052 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:25,052 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:25,053 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:28,193 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:14:30,053 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:30,053 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:35,054 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:35,054 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:40,055 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:40,055 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:45,056 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:45,056 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:46,510 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:14:50,057 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:50,057 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:14:55,057 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:14:55,058 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:00,058 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:00,058 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:04,105 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:15:05,059 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:05,059 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:10,060 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:10,060 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:15,061 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:15,061 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:20,061 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:20,062 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:21,124 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:15:25,062 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:25,063 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:30,063 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:30,063 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:35,064 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:35,064 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:37,845 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:15:40,065 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:40,065 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:45,066 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:45,067 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:50,068 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:50,068 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:15:54,852 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:15:55,069 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:15:55,069 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:00,070 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:00,070 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:05,071 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:05,072 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:10,072 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:10,073 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:12,797 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:16:15,073 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:15,074 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:20,075 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:20,075 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:25,076 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:25,076 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:30,077 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:30,077 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:30,171 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:16:35,078 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:35,078 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:40,079 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:40,079 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:45,080 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:45,081 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:48,384 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:16:50,081 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:50,082 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:16:55,082 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:16:55,083 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:00,084 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:00,084 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:05,085 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:05,085 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:05,252 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:17:10,086 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:10,086 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:15,087 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:15,087 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:20,088 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:20,089 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:22,319 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:17:25,089 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:25,090 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:30,091 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:30,091 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:35,092 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:35,092 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:39,245 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:17:40,093 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:40,093 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:45,094 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:45,094 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:50,095 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:50,095 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:55,096 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:17:55,097 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:17:56,550 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:18:00,097 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:00,098 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:05,098 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:05,099 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:10,099 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:10,100 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:14,549 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:18:15,101 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:15,101 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:20,102 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:20,102 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:25,103 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:25,103 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:30,104 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:30,104 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:31,431 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:18:35,105 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:35,105 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:40,106 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:40,106 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:45,107 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:45,107 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:48,392 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:18:50,108 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:50,108 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:18:55,109 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:18:55,109 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:00,110 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:00,110 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:05,111 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:05,112 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:05,220 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:19:10,112 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:10,113 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:15,113 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:15,114 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:20,115 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:20,115 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:22,910 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:19:25,116 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:25,116 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:30,117 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:30,117 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:35,118 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:35,119 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:40,119 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:40,120 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:40,233 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:19:45,120 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:45,121 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:50,121 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:50,122 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:55,123 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:19:55,123 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:19:57,913 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:20:00,124 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:00,124 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:05,125 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:05,126 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:10,127 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:10,127 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:15,128 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:15,128 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:16,359 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:20:20,129 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:20,129 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:25,130 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:25,130 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:30,131 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:30,131 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:33,500 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:20:35,132 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:35,132 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:40,133 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:40,133 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:45,134 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:45,134 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:50,135 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:50,135 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:20:50,760 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:20:55,136 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:20:55,136 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:00,137 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:00,137 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:05,138 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:05,138 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:07,584 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:21:10,139 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:10,139 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:15,140 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:15,140 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:20,141 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:20,141 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:24,941 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:21:25,142 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:25,142 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:30,143 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:30,143 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:35,144 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:35,144 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:40,145 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:40,146 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:42,733 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:21:45,146 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:45,147 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:50,147 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:50,148 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:21:55,148 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:21:55,149 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:00,150 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:00,150 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:00,747 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:22:05,151 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:05,151 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:10,152 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:10,153 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:15,153 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:15,154 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:18,302 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:22:20,155 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:20,155 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:25,156 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:25,156 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:30,157 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:30,157 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:35,158 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:35,159 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:35,686 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:22:40,159 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:40,160 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:45,160 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:45,161 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:50,162 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:50,162 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:22:52,455 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:22:55,163 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:22:55,163 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:00,164 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:00,164 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:05,165 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:05,165 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:09,754 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:23:10,166 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:10,166 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:15,167 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:15,167 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:20,168 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:20,168 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:25,169 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:25,169 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:27,516 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:23:30,170 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:30,170 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:35,171 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:35,171 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:40,172 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:40,172 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:45,173 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:45,173 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:45,593 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:23:50,174 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:50,174 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:23:55,175 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:23:55,175 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:00,176 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:00,176 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:02,258 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:24:05,177 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:05,177 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:10,178 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:10,178 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:15,179 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:15,179 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:19,515 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:24:20,180 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:20,180 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:25,181 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:25,181 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:30,182 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:30,183 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:35,183 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:35,184 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:36,727 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:24:40,184 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:40,185 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:45,185 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:45,186 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:50,186 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:50,187 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:24:53,849 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:24:55,187 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:24:55,188 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:00,188 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:00,189 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:05,189 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:05,190 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:10,190 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:10,191 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:12,036 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:25:15,192 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:15,192 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:20,193 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:20,193 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:25,194 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:25,194 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:29,990 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:25:30,195 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:30,195 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:35,196 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:35,196 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:40,197 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:40,197 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:45,198 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:45,198 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:50,199 DEBUG HandlerThread:31571 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:25:50,200 DEBUG SenderThread:31571 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:25:52,526 INFO memory :31571 [interfaces.py:monitor():140] Process proc.memory.rssMB has exited. +2024-05-09 11:25:52,527 DEBUG SystemMonitor:31571 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-09 11:25:52,528 DEBUG SystemMonitor:31571 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-09 11:25:53,231 WARNING StreamThr :31571 [internal.py:is_dead():414] Internal process exiting, parent pid 31356 disappeared +2024-05-09 11:25:53,231 ERROR StreamThr :31571 [internal.py:wandb_internal():152] Internal process shutdown. +2024-05-09 11:25:53,531 INFO HandlerThread:31571 [handler.py:finish():866] shutting down handler +2024-05-09 11:25:53,532 INFO HandlerThread:31571 [system_monitor.py:finish():203] Stopping system monitor +2024-05-09 11:25:53,532 INFO WriterThread:31571 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_110904-sc0bbka3/run-sc0bbka3.wandb +2024-05-09 11:25:53,532 INFO HandlerThread:31571 [interfaces.py:finish():202] Joined cpu monitor +2024-05-09 11:25:53,532 INFO HandlerThread:31571 [interfaces.py:finish():202] Joined disk monitor +2024-05-09 11:25:53,532 INFO HandlerThread:31571 [interfaces.py:finish():202] Joined gpu monitor +2024-05-09 11:25:53,533 INFO HandlerThread:31571 [interfaces.py:finish():202] Joined memory monitor +2024-05-09 11:25:53,533 INFO HandlerThread:31571 [interfaces.py:finish():202] Joined network monitor +2024-05-09 11:25:54,200 INFO SenderThread:31571 [sender.py:finish():1546] shutting down sender diff --git a/wandb/offline-run-20240509_110904-sc0bbka3/logs/debug.log b/wandb/offline-run-20240509_110904-sc0bbka3/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..162378426d716dfd3af90698285a17a83fea185d --- /dev/null +++ b/wandb/offline-run-20240509_110904-sc0bbka3/logs/debug.log @@ -0,0 +1,26 @@ +2024-05-09 11:09:04,096 INFO MainThread:31356 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-09 11:09:04,098 INFO MainThread:31356 [wandb_setup.py:_flush():76] Configure stats pid to 31356 +2024-05-09 11:09:04,098 INFO MainThread:31356 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-09 11:09:04,098 INFO MainThread:31356 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-09 11:09:04,098 INFO MainThread:31356 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-09 11:09:04,098 INFO MainThread:31356 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-09 11:09:04,098 INFO MainThread:31356 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-09 11:09:04,098 INFO MainThread:31356 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-09 11:09:04,098 INFO MainThread:31356 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-09 11:09:04,098 INFO MainThread:31356 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_110904-sc0bbka3/logs/debug.log +2024-05-09 11:09:04,099 INFO MainThread:31356 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_110904-sc0bbka3/logs/debug-internal.log +2024-05-09 11:09:04,099 INFO MainThread:31356 [wandb_init.py:init():561] calling init triggers +2024-05-09 11:09:04,099 INFO MainThread:31356 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 8, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-09 11:09:04,099 INFO MainThread:31356 [wandb_init.py:init():611] starting backend +2024-05-09 11:09:04,099 INFO MainThread:31356 [wandb_init.py:init():615] setting up manager +2024-05-09 11:09:04,109 INFO MainThread:31356 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-09 11:09:04,112 INFO MainThread:31356 [wandb_init.py:init():623] backend started and connected +2024-05-09 11:09:04,125 INFO MainThread:31356 [wandb_init.py:init():715] updated telemetry +2024-05-09 11:09:04,136 INFO MainThread:31356 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-09 11:09:04,143 INFO MainThread:31356 [wandb_init.py:init():799] starting run threads in backend +2024-05-09 11:09:11,215 INFO MainThread:31356 [wandb_run.py:_console_start():2335] atexit reg +2024-05-09 11:09:11,215 INFO MainThread:31356 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-09 11:09:11,215 INFO MainThread:31356 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-09 11:09:11,216 INFO MainThread:31356 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-09 11:09:11,216 INFO MainThread:31356 [wandb_init.py:init():842] run started, returning control to user process diff --git a/wandb/offline-run-20240509_110904-sc0bbka3/run-sc0bbka3.wandb b/wandb/offline-run-20240509_110904-sc0bbka3/run-sc0bbka3.wandb new file mode 100644 index 0000000000000000000000000000000000000000..9f2b8899c48e61255234928127be233f77392a78 --- /dev/null +++ b/wandb/offline-run-20240509_110904-sc0bbka3/run-sc0bbka3.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1238a264dadc18c1188ec74d3758b8168d1b475ce24759c4ff0a6a77d0433b26 +size 145059 diff --git a/wandb/offline-run-20240509_113410-b7nw78zg/files/conda-environment.yaml b/wandb/offline-run-20240509_113410-b7nw78zg/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ed56ab5ac90a4bed6f79ed5c79ec57e153ec12f --- /dev/null +++ b/wandb/offline-run-20240509_113410-b7nw78zg/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.37.2 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240509_113410-b7nw78zg/files/wandb-metadata.json b/wandb/offline-run-20240509_113410-b7nw78zg/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..423524eec19b9cb971ce3544e08e10867787aa73 --- /dev/null +++ b/wandb/offline-run-20240509_113410-b7nw78zg/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T18:34:10.690731", + "startedAt": "2024-05-09T18:34:10.539844", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "128", + "--model.finetune_per_device_batch_size", + "16", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-11n01.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.6109375, + "min": 1500.0, + "max": 2800.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.626354217529297 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5838279724121 + } +} diff --git a/wandb/offline-run-20240509_113410-b7nw78zg/logs/debug-internal.log b/wandb/offline-run-20240509_113410-b7nw78zg/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..c796952d95a462d3ff2e0f5355852945c1b1879e --- /dev/null +++ b/wandb/offline-run-20240509_113410-b7nw78zg/logs/debug-internal.log @@ -0,0 +1,485 @@ +2024-05-09 11:34:10,567 INFO StreamThr :34290 [internal.py:wandb_internal():86] W&B internal server running at pid: 34290, started at: 2024-05-09 11:34:10.563455 +2024-05-09 11:34:10,570 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status +2024-05-09 11:34:10,573 INFO WriterThread:34290 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_113410-b7nw78zg/run-b7nw78zg.wandb +2024-05-09 11:34:10,632 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: run_start +2024-05-09 11:34:10,649 DEBUG HandlerThread:34290 [system_info.py:__init__():26] System info init +2024-05-09 11:34:10,649 DEBUG HandlerThread:34290 [system_info.py:__init__():41] System info init done +2024-05-09 11:34:10,649 INFO HandlerThread:34290 [system_monitor.py:start():194] Starting system monitor +2024-05-09 11:34:10,649 INFO SystemMonitor:34290 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-09 11:34:10,649 INFO HandlerThread:34290 [system_monitor.py:probe():214] Collecting system info +2024-05-09 11:34:10,654 INFO SystemMonitor:34290 [interfaces.py:start():190] Started cpu monitoring +2024-05-09 11:34:10,655 INFO SystemMonitor:34290 [interfaces.py:start():190] Started disk monitoring +2024-05-09 11:34:10,657 INFO SystemMonitor:34290 [interfaces.py:start():190] Started gpu monitoring +2024-05-09 11:34:10,658 INFO SystemMonitor:34290 [interfaces.py:start():190] Started memory monitoring +2024-05-09 11:34:10,659 INFO SystemMonitor:34290 [interfaces.py:start():190] Started network monitoring +2024-05-09 11:34:10,690 DEBUG HandlerThread:34290 [system_info.py:probe():150] Probing system +2024-05-09 11:34:10,693 DEBUG HandlerThread:34290 [system_info.py:_probe_git():135] Probing git +2024-05-09 11:34:10,705 DEBUG HandlerThread:34290 [system_info.py:_probe_git():143] Probing git done +2024-05-09 11:34:10,706 DEBUG HandlerThread:34290 [system_info.py:probe():198] Probing system done +2024-05-09 11:34:10,706 DEBUG HandlerThread:34290 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T18:34:10.690731', 'startedAt': '2024-05-09T18:34:10.539844', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '128', '--model.finetune_per_device_batch_size', '16', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-11n01.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.6109375, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.626354217529297}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5838279724121}} +2024-05-09 11:34:10,706 INFO HandlerThread:34290 [system_monitor.py:probe():224] Finished collecting system info +2024-05-09 11:34:10,706 INFO HandlerThread:34290 [system_monitor.py:probe():227] Publishing system info +2024-05-09 11:34:10,706 DEBUG HandlerThread:34290 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-09 11:34:17,432 DEBUG HandlerThread:34290 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-09 11:34:17,436 INFO HandlerThread:34290 [system_monitor.py:probe():229] Finished publishing system info +2024-05-09 11:34:17,440 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:34:17,440 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: keepalive +2024-05-09 11:34:17,441 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:34:17,655 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: python_packages +2024-05-09 11:34:21,442 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:34:21,442 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:34:26,443 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:34:26,443 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:34:31,444 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:34:31,444 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:34:36,445 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:34:36,446 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:34:39,043 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:34:41,447 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:34:41,447 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:34:46,448 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:34:46,448 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:34:51,449 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:34:51,450 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:34:56,450 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:34:56,451 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:34:57,745 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:35:01,452 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:01,452 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:06,453 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:06,453 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:10,659 DEBUG SystemMonitor:34290 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-09 11:35:11,454 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:11,454 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:15,745 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:35:16,455 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:16,456 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:21,456 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:21,457 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:26,458 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:26,458 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:31,459 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:31,459 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:33,369 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:35:36,460 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:36,460 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:41,461 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:41,461 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:46,462 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:46,462 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:50,585 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:35:51,463 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:51,463 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:35:56,464 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:35:56,464 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:01,465 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:01,466 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:06,466 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:06,467 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:07,335 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:36:11,467 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:11,468 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:16,468 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:16,469 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:21,470 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:21,470 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:24,114 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:36:26,471 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:26,471 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:31,472 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:31,472 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:36,473 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:36,473 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:41,226 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:36:41,474 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:41,474 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:46,475 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:46,475 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:51,476 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:51,476 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:56,477 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:36:56,477 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:36:58,174 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:37:01,478 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:01,479 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:06,479 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:06,480 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:11,480 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:11,481 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:15,034 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:37:16,481 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:16,482 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:21,483 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:21,483 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:26,484 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:26,484 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:31,485 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:31,486 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:31,974 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:37:36,486 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:36,487 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:41,487 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:41,488 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:46,489 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:46,489 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:48,828 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:37:51,490 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:51,491 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:37:56,491 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:37:56,492 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:01,492 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:01,493 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:05,973 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:38:06,493 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:06,494 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:11,495 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:11,495 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:16,496 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:16,496 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:21,497 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:21,497 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:23,566 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:38:26,498 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:26,498 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:31,499 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:31,499 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:36,500 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:36,501 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:40,555 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:38:41,501 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:41,502 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:46,503 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:46,503 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:51,504 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:51,504 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:56,505 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:38:56,505 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:38:58,805 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:39:01,506 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:01,506 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:06,507 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:06,508 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:11,508 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:11,509 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:16,509 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:16,510 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:16,836 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:39:21,511 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:21,511 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:26,512 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:26,512 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:31,513 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:31,513 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:33,845 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:39:36,514 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:36,514 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:41,515 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:41,515 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:46,516 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:46,517 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:50,971 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:39:51,517 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:51,518 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:39:56,519 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:39:56,519 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:01,520 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:01,520 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:06,521 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:06,521 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:08,011 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:40:11,522 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:11,522 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:16,523 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:16,523 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:21,524 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:21,525 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:24,966 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:40:26,525 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:26,526 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:31,526 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:31,527 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:36,528 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:36,528 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:41,529 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:41,529 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:41,744 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:40:46,530 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:46,531 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:51,531 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:51,532 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:56,532 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:40:56,533 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:40:58,637 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:41:01,534 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:01,534 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:06,535 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:06,535 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:11,536 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:11,536 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:16,537 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:16,537 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:17,104 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:41:21,538 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:21,538 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:26,539 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:26,540 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:31,540 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:31,541 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:34,085 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:41:36,542 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:36,542 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:41,543 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:41,543 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:46,544 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:46,544 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:51,545 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:51,545 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:41:51,884 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:41:56,546 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:41:56,547 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:01,547 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:01,548 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:06,549 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:06,549 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:09,098 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:42:11,550 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:11,550 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:16,551 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:16,552 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:21,552 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:21,553 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:26,554 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:26,554 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:26,638 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:42:31,555 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:31,555 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:36,556 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:36,556 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:41,557 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:41,558 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:43,626 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:42:46,558 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:46,559 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:51,560 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:51,560 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:42:56,561 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:42:56,561 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:00,895 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:43:01,562 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:01,562 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:06,563 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:06,563 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:11,564 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:11,565 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:16,566 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:16,566 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:17,857 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:43:21,567 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:21,567 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:26,568 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:26,568 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:31,569 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:31,569 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:34,686 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:43:36,570 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:36,571 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:41,571 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:41,572 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:46,573 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:46,573 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:51,566 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:43:51,574 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:51,574 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:43:56,575 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:43:56,575 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:01,576 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:01,576 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:06,577 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:06,578 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:08,510 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:44:11,578 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:11,579 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:16,579 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:16,580 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:21,581 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:21,581 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:26,347 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:44:26,582 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:26,582 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:31,583 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:31,583 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:36,584 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:36,584 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:41,585 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:41,586 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:43,370 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:44:46,586 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:46,587 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:51,588 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:51,588 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:44:56,589 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:44:56,589 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:00,548 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:45:01,591 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:01,591 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:06,592 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:06,592 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:11,593 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:11,593 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:16,594 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:16,595 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:18,787 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:45:21,596 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:21,596 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:26,597 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:26,597 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:31,598 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:31,598 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:36,374 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:45:36,599 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:36,599 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:41,600 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:41,600 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:46,601 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:46,601 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:51,602 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:51,603 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:45:53,947 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:45:56,603 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:45:56,604 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:01,605 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:01,605 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:06,606 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:06,606 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:10,773 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:46:11,607 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:11,607 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:16,608 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:16,608 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:21,609 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:21,610 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:26,612 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:26,613 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:28,070 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:46:31,614 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:31,614 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:36,615 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:36,615 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:41,616 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:41,616 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:45,335 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:46:46,617 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:46,618 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:51,618 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:51,619 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:46:56,620 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:46:56,620 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:01,621 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:01,621 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:02,517 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:47:06,622 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:06,622 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:11,623 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:11,623 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:16,624 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:16,624 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:20,302 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:47:21,625 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:21,626 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:26,626 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:26,627 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:31,628 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:31,628 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:36,629 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:36,629 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:38,073 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:47:41,630 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:41,630 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:46,631 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:46,631 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:51,632 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:51,632 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:47:54,740 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:47:56,633 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:47:56,634 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:01,634 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:01,635 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:06,636 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:06,636 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:11,637 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:11,637 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:12,276 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:48:16,638 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:16,638 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:21,639 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:21,640 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:26,641 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:26,641 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:30,745 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:48:31,642 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:31,642 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:36,643 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:36,643 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:41,644 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:41,645 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:46,645 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:46,646 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:48,416 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:48:51,647 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:51,647 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:48:56,648 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:48:56,648 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:01,649 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:01,649 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:05,364 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:49:06,651 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:06,651 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:11,652 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:11,652 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:16,654 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:16,654 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:21,655 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:21,656 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:23,341 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:49:26,657 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:26,657 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:31,658 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:31,659 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:36,660 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:36,660 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:40,384 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:49:41,661 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:41,661 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:46,663 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:46,663 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:51,664 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:51,664 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:56,665 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:49:56,666 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:49:57,422 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:50:01,667 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:50:01,667 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:50:06,668 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:50:06,669 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:50:11,670 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:50:11,670 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:50:16,079 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:50:16,671 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:50:16,672 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:50:21,673 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:50:21,673 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:50:26,674 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:50:26,675 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:50:31,676 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:50:31,676 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:50:33,136 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 11:50:36,677 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:50:36,677 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:50:41,678 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:50:41,679 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:50:46,683 DEBUG HandlerThread:34290 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 11:50:46,687 DEBUG SenderThread:34290 [sender.py:send_request():406] send_request: status_report +2024-05-09 11:50:48,923 INFO memory :34290 [interfaces.py:monitor():140] Process proc.memory.rssMB has exited. +2024-05-09 11:50:48,924 DEBUG SystemMonitor:34290 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-09 11:50:48,925 DEBUG SystemMonitor:34290 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-09 11:50:49,762 INFO MainThread:34290 [internal.py:handle_exit():76] Internal process exited diff --git a/wandb/offline-run-20240509_113410-b7nw78zg/logs/debug.log b/wandb/offline-run-20240509_113410-b7nw78zg/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0a85622fccf73408e5ed3bf63fcc5eed0cc3ed0d --- /dev/null +++ b/wandb/offline-run-20240509_113410-b7nw78zg/logs/debug.log @@ -0,0 +1,26 @@ +2024-05-09 11:34:10,552 INFO MainThread:34084 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-09 11:34:10,555 INFO MainThread:34084 [wandb_setup.py:_flush():76] Configure stats pid to 34084 +2024-05-09 11:34:10,555 INFO MainThread:34084 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_113410-b7nw78zg/logs/debug.log +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_113410-b7nw78zg/logs/debug-internal.log +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_init.py:init():561] calling init triggers +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_init.py:init():611] starting backend +2024-05-09 11:34:10,556 INFO MainThread:34084 [wandb_init.py:init():615] setting up manager +2024-05-09 11:34:10,561 INFO MainThread:34084 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-09 11:34:10,564 INFO MainThread:34084 [wandb_init.py:init():623] backend started and connected +2024-05-09 11:34:10,572 INFO MainThread:34084 [wandb_init.py:init():715] updated telemetry +2024-05-09 11:34:10,580 INFO MainThread:34084 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-09 11:34:10,587 INFO MainThread:34084 [wandb_init.py:init():799] starting run threads in backend +2024-05-09 11:34:17,654 INFO MainThread:34084 [wandb_run.py:_console_start():2335] atexit reg +2024-05-09 11:34:17,655 INFO MainThread:34084 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-09 11:34:17,655 INFO MainThread:34084 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-09 11:34:17,655 INFO MainThread:34084 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-09 11:34:17,655 INFO MainThread:34084 [wandb_init.py:init():842] run started, returning control to user process diff --git a/wandb/offline-run-20240509_113410-b7nw78zg/run-b7nw78zg.wandb b/wandb/offline-run-20240509_113410-b7nw78zg/run-b7nw78zg.wandb new file mode 100644 index 0000000000000000000000000000000000000000..5f9da3f8612054767ae4f339fe05a5331c97dd93 --- /dev/null +++ b/wandb/offline-run-20240509_113410-b7nw78zg/run-b7nw78zg.wandb @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c1e06f3cd24aead00a0870e5922f420bf4b7ddda4715e1d4f95d465060498835 +size 133438 diff --git a/wandb/offline-run-20240509_133710-isri379u/files/conda-environment.yaml b/wandb/offline-run-20240509_133710-isri379u/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8ed56ab5ac90a4bed6f79ed5c79ec57e153ec12f --- /dev/null +++ b/wandb/offline-run-20240509_133710-isri379u/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.37.2 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240509_133710-isri379u/files/wandb-metadata.json b/wandb/offline-run-20240509_133710-isri379u/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b3f112a718f356c411363d9fa49c53661f1443e2 --- /dev/null +++ b/wandb/offline-run-20240509_133710-isri379u/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T20:37:10.416774", + "startedAt": "2024-05-09T20:37:10.271022", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "128", + "--model.finetune_per_device_batch_size", + "16", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-11n01.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 2.492187500000001, + "min": 1500.0, + "max": 2800.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.6262054443359375 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5838279724121 + } +} diff --git a/wandb/offline-run-20240509_133710-isri379u/files/wandb-summary.json b/wandb/offline-run-20240509_133710-isri379u/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..dc57bc6a43340cee0721003bff05ae64a4425055 --- /dev/null +++ b/wandb/offline-run-20240509_133710-isri379u/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 116}} \ No newline at end of file diff --git a/wandb/offline-run-20240509_133710-isri379u/logs/debug-internal.log b/wandb/offline-run-20240509_133710-isri379u/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..32dde3bfd6471e7ca0ccd8a0a1b4ce185d47bb41 --- /dev/null +++ b/wandb/offline-run-20240509_133710-isri379u/logs/debug-internal.log @@ -0,0 +1,183 @@ +2024-05-09 13:37:10,292 INFO StreamThr :842 [internal.py:wandb_internal():86] W&B internal server running at pid: 842, started at: 2024-05-09 13:37:10.290403 +2024-05-09 13:37:10,295 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status +2024-05-09 13:37:10,301 INFO WriterThread:842 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_133710-isri379u/run-isri379u.wandb +2024-05-09 13:37:10,370 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: run_start +2024-05-09 13:37:10,387 DEBUG HandlerThread:842 [system_info.py:__init__():26] System info init +2024-05-09 13:37:10,387 DEBUG HandlerThread:842 [system_info.py:__init__():41] System info init done +2024-05-09 13:37:10,387 INFO HandlerThread:842 [system_monitor.py:start():194] Starting system monitor +2024-05-09 13:37:10,387 INFO SystemMonitor:842 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-09 13:37:10,388 INFO HandlerThread:842 [system_monitor.py:probe():214] Collecting system info +2024-05-09 13:37:10,388 INFO SystemMonitor:842 [interfaces.py:start():190] Started cpu monitoring +2024-05-09 13:37:10,389 INFO SystemMonitor:842 [interfaces.py:start():190] Started disk monitoring +2024-05-09 13:37:10,391 INFO SystemMonitor:842 [interfaces.py:start():190] Started gpu monitoring +2024-05-09 13:37:10,392 INFO SystemMonitor:842 [interfaces.py:start():190] Started memory monitoring +2024-05-09 13:37:10,392 INFO SystemMonitor:842 [interfaces.py:start():190] Started network monitoring +2024-05-09 13:37:10,416 DEBUG HandlerThread:842 [system_info.py:probe():150] Probing system +2024-05-09 13:37:10,418 DEBUG HandlerThread:842 [system_info.py:_probe_git():135] Probing git +2024-05-09 13:37:10,427 DEBUG HandlerThread:842 [system_info.py:_probe_git():143] Probing git done +2024-05-09 13:37:10,427 DEBUG HandlerThread:842 [system_info.py:probe():198] Probing system done +2024-05-09 13:37:10,427 DEBUG HandlerThread:842 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T20:37:10.416774', 'startedAt': '2024-05-09T20:37:10.271022', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '128', '--model.finetune_per_device_batch_size', '16', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-11n01.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 2.492187500000001, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.6262054443359375}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5838279724121}} +2024-05-09 13:37:10,427 INFO HandlerThread:842 [system_monitor.py:probe():224] Finished collecting system info +2024-05-09 13:37:10,427 INFO HandlerThread:842 [system_monitor.py:probe():227] Publishing system info +2024-05-09 13:37:10,427 DEBUG HandlerThread:842 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-09 13:37:18,905 DEBUG HandlerThread:842 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-09 13:37:18,908 INFO HandlerThread:842 [system_monitor.py:probe():229] Finished publishing system info +2024-05-09 13:37:18,917 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:37:18,918 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: keepalive +2024-05-09 13:37:18,918 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:37:19,844 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: python_packages +2024-05-09 13:37:20,919 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:37:20,919 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:37:25,920 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:37:25,921 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:37:30,922 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:37:30,922 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:37:35,923 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:37:35,923 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:37:40,924 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:37:40,924 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:37:45,432 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:37:45,925 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:37:45,926 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:37:50,927 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:37:50,927 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:37:55,928 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:37:55,928 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:00,929 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:00,930 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:04,530 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:38:05,930 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:05,931 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:10,393 DEBUG SystemMonitor:842 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-09 13:38:10,932 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:10,932 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:15,933 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:15,933 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:20,934 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:20,934 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:22,772 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:38:25,935 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:25,935 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:30,936 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:30,936 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:35,937 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:35,937 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:40,458 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:38:40,938 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:40,939 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:45,940 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:45,940 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:50,941 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:50,941 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:55,942 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:38:55,942 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:38:57,723 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:39:00,943 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:39:00,943 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:39:05,944 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:39:05,945 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:39:07,280 DEBUG SenderThread:842 [sender.py:send():379] send: exit +2024-05-09 13:39:07,280 INFO SenderThread:842 [sender.py:send_exit():586] handling exit code: 1 +2024-05-09 13:39:07,280 INFO SenderThread:842 [sender.py:send_exit():588] handling runtime: 116 +2024-05-09 13:39:07,282 INFO SenderThread:842 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-09 13:39:07,282 INFO SenderThread:842 [sender.py:send_exit():594] send defer +2024-05-09 13:39:07,283 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,283 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 0 +2024-05-09 13:39:07,283 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,283 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-05-09 13:39:07,283 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 1 +2024-05-09 13:39:07,283 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,283 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 1 +2024-05-09 13:39:07,284 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,284 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-05-09 13:39:07,284 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 2 +2024-05-09 13:39:07,284 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,284 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 2 +2024-05-09 13:39:07,284 INFO HandlerThread:842 [system_monitor.py:finish():203] Stopping system monitor +2024-05-09 13:39:07,284 DEBUG SystemMonitor:842 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-09 13:39:07,285 DEBUG SystemMonitor:842 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-09 13:39:07,285 INFO HandlerThread:842 [interfaces.py:finish():202] Joined cpu monitor +2024-05-09 13:39:07,286 INFO HandlerThread:842 [interfaces.py:finish():202] Joined disk monitor +2024-05-09 13:39:07,315 INFO HandlerThread:842 [interfaces.py:finish():202] Joined gpu monitor +2024-05-09 13:39:07,316 INFO HandlerThread:842 [interfaces.py:finish():202] Joined memory monitor +2024-05-09 13:39:07,316 INFO HandlerThread:842 [interfaces.py:finish():202] Joined network monitor +2024-05-09 13:39:07,316 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,316 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-05-09 13:39:07,316 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 3 +2024-05-09 13:39:07,317 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,317 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 3 +2024-05-09 13:39:07,317 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,317 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-05-09 13:39:07,317 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 4 +2024-05-09 13:39:07,318 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,318 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 4 +2024-05-09 13:39:07,318 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,318 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-05-09 13:39:07,318 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 5 +2024-05-09 13:39:07,318 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,318 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 5 +2024-05-09 13:39:07,318 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,319 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-05-09 13:39:07,319 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 6 +2024-05-09 13:39:07,319 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,319 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 6 +2024-05-09 13:39:07,319 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,319 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-05-09 13:39:07,319 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 7 +2024-05-09 13:39:07,319 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:39:07,319 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,319 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:39:07,320 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 7 +2024-05-09 13:39:07,320 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,320 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-05-09 13:39:07,320 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 8 +2024-05-09 13:39:07,320 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,320 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 8 +2024-05-09 13:39:07,320 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,320 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-05-09 13:39:07,320 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 9 +2024-05-09 13:39:07,320 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,321 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 9 +2024-05-09 13:39:07,321 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,321 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-05-09 13:39:07,321 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 10 +2024-05-09 13:39:07,321 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,321 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 10 +2024-05-09 13:39:07,321 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,321 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-05-09 13:39:07,321 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 11 +2024-05-09 13:39:07,321 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,321 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 11 +2024-05-09 13:39:07,321 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,322 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-05-09 13:39:07,322 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 12 +2024-05-09 13:39:07,322 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,322 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 12 +2024-05-09 13:39:07,322 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,322 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-05-09 13:39:07,322 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 13 +2024-05-09 13:39:07,322 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,322 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 13 +2024-05-09 13:39:07,322 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,322 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-05-09 13:39:07,322 INFO SenderThread:842 [sender.py:transition_state():614] send defer: 14 +2024-05-09 13:39:07,323 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:39:07,323 DEBUG SenderThread:842 [sender.py:send():379] send: final +2024-05-09 13:39:07,323 INFO HandlerThread:842 [handler.py:handle_request_defer():172] handle defer: 14 +2024-05-09 13:39:07,323 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: defer +2024-05-09 13:39:07,323 INFO SenderThread:842 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-05-09 13:39:07,324 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-09 13:39:07,324 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-09 13:39:07,324 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: poll_exit +2024-05-09 13:39:07,324 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: poll_exit +2024-05-09 13:39:07,324 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: server_info +2024-05-09 13:39:07,325 DEBUG SenderThread:842 [sender.py:send_request():406] send_request: server_info +2024-05-09 13:39:07,325 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: get_summary +2024-05-09 13:39:07,325 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: sampled_history +2024-05-09 13:39:07,326 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-09 13:39:07,326 INFO MainThread:842 [wandb_run.py:_footer_history_summary_info():3936] rendering history +2024-05-09 13:39:07,326 INFO MainThread:842 [wandb_run.py:_footer_history_summary_info():3968] rendering summary +2024-05-09 13:39:07,327 DEBUG HandlerThread:842 [handler.py:handle_request():146] handle_request: shutdown +2024-05-09 13:39:07,327 INFO HandlerThread:842 [handler.py:finish():866] shutting down handler +2024-05-09 13:39:08,325 INFO WriterThread:842 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_133710-isri379u/run-isri379u.wandb +2024-05-09 13:39:08,325 INFO SenderThread:842 [sender.py:finish():1546] shutting down sender diff --git a/wandb/offline-run-20240509_133710-isri379u/logs/debug.log b/wandb/offline-run-20240509_133710-isri379u/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..c50058eb1ec1564140a813fe4085aebc8a491245 --- /dev/null +++ b/wandb/offline-run-20240509_133710-isri379u/logs/debug.log @@ -0,0 +1,27 @@ +2024-05-09 13:37:10,281 INFO MainThread:485 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_setup.py:_flush():76] Configure stats pid to 485 +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_133710-isri379u/logs/debug.log +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_133710-isri379u/logs/debug-internal.log +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_init.py:init():561] calling init triggers +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_init.py:init():611] starting backend +2024-05-09 13:37:10,283 INFO MainThread:485 [wandb_init.py:init():615] setting up manager +2024-05-09 13:37:10,288 INFO MainThread:485 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-09 13:37:10,293 INFO MainThread:485 [wandb_init.py:init():623] backend started and connected +2024-05-09 13:37:10,301 INFO MainThread:485 [wandb_init.py:init():715] updated telemetry +2024-05-09 13:37:10,321 INFO MainThread:485 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-09 13:37:10,326 INFO MainThread:485 [wandb_init.py:init():799] starting run threads in backend +2024-05-09 13:37:19,844 INFO MainThread:485 [wandb_run.py:_console_start():2335] atexit reg +2024-05-09 13:37:19,844 INFO MainThread:485 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-09 13:37:19,844 INFO MainThread:485 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-09 13:37:19,844 INFO MainThread:485 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-09 13:37:19,845 INFO MainThread:485 [wandb_init.py:init():842] run started, returning control to user process +2024-05-09 13:39:08,328 WARNING MsgRouterThr:485 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/offline-run-20240509_133710-isri379u/run-isri379u.wandb b/wandb/offline-run-20240509_133710-isri379u/run-isri379u.wandb new file mode 100644 index 0000000000000000000000000000000000000000..ec3e4ae43b010d9717c89f7764a515f529f93dd2 Binary files /dev/null and b/wandb/offline-run-20240509_133710-isri379u/run-isri379u.wandb differ diff --git a/wandb/offline-run-20240509_135023-aa8ujb3v/files/conda-environment.yaml b/wandb/offline-run-20240509_135023-aa8ujb3v/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb5c46cd8a524560371e6601fcdbf9ef63efe33a --- /dev/null +++ b/wandb/offline-run-20240509_135023-aa8ujb3v/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.40.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240509_135023-aa8ujb3v/files/wandb-metadata.json b/wandb/offline-run-20240509_135023-aa8ujb3v/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..31c7bff7eaf97a071c05ba0e968c37920c46e717 --- /dev/null +++ b/wandb/offline-run-20240509_135023-aa8ujb3v/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T20:50:24.103790", + "startedAt": "2024-05-09T20:50:23.961841", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "128", + "--model.finetune_per_device_batch_size", + "16", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-11n01.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 2.2109375, + "min": 1500.0, + "max": 2800.0 + }, + "cpu_freq_per_core": [ + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.626178741455078 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5838279724121 + } +} diff --git a/wandb/offline-run-20240509_135023-aa8ujb3v/logs/debug-internal.log b/wandb/offline-run-20240509_135023-aa8ujb3v/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..08c170234f983f8a2ea6d6ba6f4a436314f62d93 --- /dev/null +++ b/wandb/offline-run-20240509_135023-aa8ujb3v/logs/debug-internal.log @@ -0,0 +1,97 @@ +2024-05-09 13:50:23,986 INFO StreamThr :2438 [internal.py:wandb_internal():86] W&B internal server running at pid: 2438, started at: 2024-05-09 13:50:23.984008 +2024-05-09 13:50:23,988 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status +2024-05-09 13:50:23,996 INFO WriterThread:2438 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_135023-aa8ujb3v/run-aa8ujb3v.wandb +2024-05-09 13:50:24,055 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: run_start +2024-05-09 13:50:24,074 DEBUG HandlerThread:2438 [system_info.py:__init__():26] System info init +2024-05-09 13:50:24,074 DEBUG HandlerThread:2438 [system_info.py:__init__():41] System info init done +2024-05-09 13:50:24,074 INFO HandlerThread:2438 [system_monitor.py:start():194] Starting system monitor +2024-05-09 13:50:24,074 INFO SystemMonitor:2438 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-09 13:50:24,074 INFO HandlerThread:2438 [system_monitor.py:probe():214] Collecting system info +2024-05-09 13:50:24,075 INFO SystemMonitor:2438 [interfaces.py:start():190] Started cpu monitoring +2024-05-09 13:50:24,075 INFO SystemMonitor:2438 [interfaces.py:start():190] Started disk monitoring +2024-05-09 13:50:24,076 INFO SystemMonitor:2438 [interfaces.py:start():190] Started gpu monitoring +2024-05-09 13:50:24,076 INFO SystemMonitor:2438 [interfaces.py:start():190] Started memory monitoring +2024-05-09 13:50:24,077 INFO SystemMonitor:2438 [interfaces.py:start():190] Started network monitoring +2024-05-09 13:50:24,103 DEBUG HandlerThread:2438 [system_info.py:probe():150] Probing system +2024-05-09 13:50:24,107 DEBUG HandlerThread:2438 [system_info.py:_probe_git():135] Probing git +2024-05-09 13:50:24,120 DEBUG HandlerThread:2438 [system_info.py:_probe_git():143] Probing git done +2024-05-09 13:50:24,120 DEBUG HandlerThread:2438 [system_info.py:probe():198] Probing system done +2024-05-09 13:50:24,120 DEBUG HandlerThread:2438 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T20:50:24.103790', 'startedAt': '2024-05-09T20:50:23.961841', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '128', '--model.finetune_per_device_batch_size', '16', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-11n01.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 2.2109375, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.626178741455078}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5838279724121}} +2024-05-09 13:50:24,120 INFO HandlerThread:2438 [system_monitor.py:probe():224] Finished collecting system info +2024-05-09 13:50:24,120 INFO HandlerThread:2438 [system_monitor.py:probe():227] Publishing system info +2024-05-09 13:50:24,120 DEBUG HandlerThread:2438 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-09 13:50:30,744 DEBUG HandlerThread:2438 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-09 13:50:30,747 INFO HandlerThread:2438 [system_monitor.py:probe():229] Finished publishing system info +2024-05-09 13:50:30,751 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:50:30,751 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: keepalive +2024-05-09 13:50:30,752 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:50:30,989 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: python_packages +2024-05-09 13:50:34,753 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:50:34,753 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:50:39,754 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:50:39,755 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:50:44,756 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:50:44,756 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:50:49,757 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:50:49,758 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:50:53,313 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:50:54,759 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:50:54,759 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:50:59,761 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:50:59,761 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:04,762 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:04,762 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:09,763 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:09,764 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:12,224 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:51:14,765 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:14,766 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:19,767 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:19,767 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:24,077 DEBUG SystemMonitor:2438 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-09 13:51:24,768 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:24,769 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:29,770 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:29,770 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:30,352 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:51:34,771 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:34,772 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:39,773 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:39,773 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:44,774 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:44,775 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:48,020 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:51:49,776 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:49,776 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:54,777 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:54,778 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:51:59,779 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:51:59,779 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:52:04,780 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:52:04,781 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:52:05,424 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:52:09,782 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:52:09,782 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:52:14,783 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:52:14,783 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:52:19,784 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:52:19,785 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:52:22,222 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:52:24,786 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:52:24,786 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:52:29,787 DEBUG HandlerThread:2438 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:52:29,788 DEBUG SenderThread:2438 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:52:32,141 INFO memory :2438 [interfaces.py:monitor():140] Process proc.memory.rssMB has exited. +2024-05-09 13:52:32,141 DEBUG SystemMonitor:2438 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-09 13:52:32,142 DEBUG SystemMonitor:2438 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-09 13:52:33,004 WARNING StreamThr :2438 [internal.py:is_dead():414] Internal process exiting, parent pid 2212 disappeared +2024-05-09 13:52:33,004 ERROR StreamThr :2438 [internal.py:wandb_internal():152] Internal process shutdown. +2024-05-09 13:52:33,143 INFO HandlerThread:2438 [handler.py:finish():866] shutting down handler +2024-05-09 13:52:33,144 INFO WriterThread:2438 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_135023-aa8ujb3v/run-aa8ujb3v.wandb +2024-05-09 13:52:33,144 INFO HandlerThread:2438 [system_monitor.py:finish():203] Stopping system monitor +2024-05-09 13:52:33,145 INFO HandlerThread:2438 [interfaces.py:finish():202] Joined cpu monitor +2024-05-09 13:52:33,145 INFO HandlerThread:2438 [interfaces.py:finish():202] Joined disk monitor +2024-05-09 13:52:33,145 INFO HandlerThread:2438 [interfaces.py:finish():202] Joined gpu monitor +2024-05-09 13:52:33,145 INFO HandlerThread:2438 [interfaces.py:finish():202] Joined memory monitor +2024-05-09 13:52:33,145 INFO HandlerThread:2438 [interfaces.py:finish():202] Joined network monitor +2024-05-09 13:52:33,788 INFO SenderThread:2438 [sender.py:finish():1546] shutting down sender diff --git a/wandb/offline-run-20240509_135023-aa8ujb3v/logs/debug.log b/wandb/offline-run-20240509_135023-aa8ujb3v/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..9c755cbc76b5813f49bc04640be74cefa3d30b1f --- /dev/null +++ b/wandb/offline-run-20240509_135023-aa8ujb3v/logs/debug.log @@ -0,0 +1,26 @@ +2024-05-09 13:50:23,974 INFO MainThread:2212 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_setup.py:_flush():76] Configure stats pid to 2212 +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_135023-aa8ujb3v/logs/debug.log +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_135023-aa8ujb3v/logs/debug-internal.log +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_init.py:init():561] calling init triggers +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-09 13:50:23,975 INFO MainThread:2212 [wandb_init.py:init():611] starting backend +2024-05-09 13:50:23,976 INFO MainThread:2212 [wandb_init.py:init():615] setting up manager +2024-05-09 13:50:23,981 INFO MainThread:2212 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-09 13:50:23,984 INFO MainThread:2212 [wandb_init.py:init():623] backend started and connected +2024-05-09 13:50:23,995 INFO MainThread:2212 [wandb_init.py:init():715] updated telemetry +2024-05-09 13:50:24,003 INFO MainThread:2212 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-09 13:50:24,011 INFO MainThread:2212 [wandb_init.py:init():799] starting run threads in backend +2024-05-09 13:50:30,989 INFO MainThread:2212 [wandb_run.py:_console_start():2335] atexit reg +2024-05-09 13:50:30,989 INFO MainThread:2212 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-09 13:50:30,990 INFO MainThread:2212 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-09 13:50:30,990 INFO MainThread:2212 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-09 13:50:30,991 INFO MainThread:2212 [wandb_init.py:init():842] run started, returning control to user process diff --git a/wandb/offline-run-20240509_135023-aa8ujb3v/run-aa8ujb3v.wandb b/wandb/offline-run-20240509_135023-aa8ujb3v/run-aa8ujb3v.wandb new file mode 100644 index 0000000000000000000000000000000000000000..b3c0e3979ae6d5bde73abe588634d53f54324c4d Binary files /dev/null and b/wandb/offline-run-20240509_135023-aa8ujb3v/run-aa8ujb3v.wandb differ diff --git a/wandb/offline-run-20240509_135532-o84u1qny/files/conda-environment.yaml b/wandb/offline-run-20240509_135532-o84u1qny/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb5c46cd8a524560371e6601fcdbf9ef63efe33a --- /dev/null +++ b/wandb/offline-run-20240509_135532-o84u1qny/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.40.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240509_135532-o84u1qny/files/wandb-metadata.json b/wandb/offline-run-20240509_135532-o84u1qny/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..4821d8699c79452d4be666bcb8838f855f97ac61 --- /dev/null +++ b/wandb/offline-run-20240509_135532-o84u1qny/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T20:55:32.729330", + "startedAt": "2024-05-09T20:55:32.573132", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "128", + "--model.finetune_per_device_batch_size", + "32", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-11n01.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 2.2296875, + "min": 1500.0, + "max": 2800.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.626178741455078 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5838279724121 + } +} diff --git a/wandb/offline-run-20240509_135532-o84u1qny/files/wandb-summary.json b/wandb/offline-run-20240509_135532-o84u1qny/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..293683cb141ef30cc06209dda6fd72a2e21dfb88 --- /dev/null +++ b/wandb/offline-run-20240509_135532-o84u1qny/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 37}} \ No newline at end of file diff --git a/wandb/offline-run-20240509_135532-o84u1qny/logs/debug-internal.log b/wandb/offline-run-20240509_135532-o84u1qny/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..4cadad70c4a8a8072e16859d1579afdfe0388c97 --- /dev/null +++ b/wandb/offline-run-20240509_135532-o84u1qny/logs/debug-internal.log @@ -0,0 +1,147 @@ +2024-05-09 13:55:32,603 INFO StreamThr :3703 [internal.py:wandb_internal():86] W&B internal server running at pid: 3703, started at: 2024-05-09 13:55:32.601056 +2024-05-09 13:55:32,605 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: status +2024-05-09 13:55:32,609 INFO WriterThread:3703 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_135532-o84u1qny/run-o84u1qny.wandb +2024-05-09 13:55:32,675 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: run_start +2024-05-09 13:55:32,696 DEBUG HandlerThread:3703 [system_info.py:__init__():26] System info init +2024-05-09 13:55:32,696 DEBUG HandlerThread:3703 [system_info.py:__init__():41] System info init done +2024-05-09 13:55:32,696 INFO HandlerThread:3703 [system_monitor.py:start():194] Starting system monitor +2024-05-09 13:55:32,696 INFO SystemMonitor:3703 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-09 13:55:32,697 INFO HandlerThread:3703 [system_monitor.py:probe():214] Collecting system info +2024-05-09 13:55:32,697 INFO SystemMonitor:3703 [interfaces.py:start():190] Started cpu monitoring +2024-05-09 13:55:32,698 INFO SystemMonitor:3703 [interfaces.py:start():190] Started disk monitoring +2024-05-09 13:55:32,699 INFO SystemMonitor:3703 [interfaces.py:start():190] Started gpu monitoring +2024-05-09 13:55:32,699 INFO SystemMonitor:3703 [interfaces.py:start():190] Started memory monitoring +2024-05-09 13:55:32,700 INFO SystemMonitor:3703 [interfaces.py:start():190] Started network monitoring +2024-05-09 13:55:32,729 DEBUG HandlerThread:3703 [system_info.py:probe():150] Probing system +2024-05-09 13:55:32,735 DEBUG HandlerThread:3703 [system_info.py:_probe_git():135] Probing git +2024-05-09 13:55:32,748 DEBUG HandlerThread:3703 [system_info.py:_probe_git():143] Probing git done +2024-05-09 13:55:32,748 DEBUG HandlerThread:3703 [system_info.py:probe():198] Probing system done +2024-05-09 13:55:32,748 DEBUG HandlerThread:3703 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T20:55:32.729330', 'startedAt': '2024-05-09T20:55:32.573132', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '128', '--model.finetune_per_device_batch_size', '32', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-11n01.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 2.2296875, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.626178741455078}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5838279724121}} +2024-05-09 13:55:32,748 INFO HandlerThread:3703 [system_monitor.py:probe():224] Finished collecting system info +2024-05-09 13:55:32,749 INFO HandlerThread:3703 [system_monitor.py:probe():227] Publishing system info +2024-05-09 13:55:32,749 DEBUG HandlerThread:3703 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-09 13:55:39,527 DEBUG HandlerThread:3703 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-09 13:55:39,530 INFO HandlerThread:3703 [system_monitor.py:probe():229] Finished publishing system info +2024-05-09 13:55:39,535 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:55:39,535 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: keepalive +2024-05-09 13:55:39,535 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:55:39,764 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: python_packages +2024-05-09 13:55:43,536 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:55:43,536 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:55:48,537 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:55:48,537 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:55:53,538 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:55:53,539 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:55:58,539 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:55:58,540 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:56:03,540 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:56:03,541 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:56:04,850 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 13:56:08,542 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:56:08,542 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:56:10,239 DEBUG SenderThread:3703 [sender.py:send():379] send: exit +2024-05-09 13:56:10,239 INFO SenderThread:3703 [sender.py:send_exit():586] handling exit code: 1 +2024-05-09 13:56:10,239 INFO SenderThread:3703 [sender.py:send_exit():588] handling runtime: 37 +2024-05-09 13:56:10,241 INFO SenderThread:3703 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-09 13:56:10,241 INFO SenderThread:3703 [sender.py:send_exit():594] send defer +2024-05-09 13:56:10,241 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,242 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 0 +2024-05-09 13:56:10,242 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,242 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-05-09 13:56:10,242 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 1 +2024-05-09 13:56:10,242 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,242 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 1 +2024-05-09 13:56:10,242 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,242 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-05-09 13:56:10,242 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 2 +2024-05-09 13:56:10,242 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,242 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 2 +2024-05-09 13:56:10,242 INFO HandlerThread:3703 [system_monitor.py:finish():203] Stopping system monitor +2024-05-09 13:56:10,242 DEBUG SystemMonitor:3703 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-09 13:56:10,243 DEBUG SystemMonitor:3703 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-09 13:56:10,243 DEBUG SystemMonitor:3703 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-09 13:56:10,243 INFO HandlerThread:3703 [interfaces.py:finish():202] Joined cpu monitor +2024-05-09 13:56:10,244 INFO HandlerThread:3703 [interfaces.py:finish():202] Joined disk monitor +2024-05-09 13:56:10,268 INFO HandlerThread:3703 [interfaces.py:finish():202] Joined gpu monitor +2024-05-09 13:56:10,269 INFO HandlerThread:3703 [interfaces.py:finish():202] Joined memory monitor +2024-05-09 13:56:10,269 INFO HandlerThread:3703 [interfaces.py:finish():202] Joined network monitor +2024-05-09 13:56:10,269 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,269 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-05-09 13:56:10,269 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 3 +2024-05-09 13:56:10,269 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,269 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 3 +2024-05-09 13:56:10,270 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,270 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-05-09 13:56:10,270 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 4 +2024-05-09 13:56:10,270 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,270 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 4 +2024-05-09 13:56:10,270 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,270 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-05-09 13:56:10,270 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 5 +2024-05-09 13:56:10,270 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,270 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 5 +2024-05-09 13:56:10,271 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,271 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-05-09 13:56:10,271 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 6 +2024-05-09 13:56:10,271 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,271 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 6 +2024-05-09 13:56:10,271 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,271 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-05-09 13:56:10,271 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 7 +2024-05-09 13:56:10,271 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 13:56:10,271 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,271 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 7 +2024-05-09 13:56:10,271 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: status_report +2024-05-09 13:56:10,271 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,271 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-05-09 13:56:10,271 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 8 +2024-05-09 13:56:10,271 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,271 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 8 +2024-05-09 13:56:10,271 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,271 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-05-09 13:56:10,272 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 9 +2024-05-09 13:56:10,272 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,272 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 9 +2024-05-09 13:56:10,272 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,272 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-05-09 13:56:10,272 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 10 +2024-05-09 13:56:10,272 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,272 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 10 +2024-05-09 13:56:10,272 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,272 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-05-09 13:56:10,272 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 11 +2024-05-09 13:56:10,272 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,272 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 11 +2024-05-09 13:56:10,272 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,272 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-05-09 13:56:10,272 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 12 +2024-05-09 13:56:10,272 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,272 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 12 +2024-05-09 13:56:10,272 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,272 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-05-09 13:56:10,272 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 13 +2024-05-09 13:56:10,272 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,272 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 13 +2024-05-09 13:56:10,273 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,273 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-05-09 13:56:10,273 INFO SenderThread:3703 [sender.py:transition_state():614] send defer: 14 +2024-05-09 13:56:10,273 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: defer +2024-05-09 13:56:10,273 DEBUG SenderThread:3703 [sender.py:send():379] send: final +2024-05-09 13:56:10,273 INFO HandlerThread:3703 [handler.py:handle_request_defer():172] handle defer: 14 +2024-05-09 13:56:10,273 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: defer +2024-05-09 13:56:10,273 INFO SenderThread:3703 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-05-09 13:56:10,273 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-09 13:56:10,273 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: poll_exit +2024-05-09 13:56:10,273 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-09 13:56:10,274 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: poll_exit +2024-05-09 13:56:10,274 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: server_info +2024-05-09 13:56:10,274 DEBUG SenderThread:3703 [sender.py:send_request():406] send_request: server_info +2024-05-09 13:56:10,274 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: get_summary +2024-05-09 13:56:10,274 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: sampled_history +2024-05-09 13:56:10,274 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-09 13:56:10,275 INFO MainThread:3703 [wandb_run.py:_footer_history_summary_info():3936] rendering history +2024-05-09 13:56:10,275 INFO MainThread:3703 [wandb_run.py:_footer_history_summary_info():3968] rendering summary +2024-05-09 13:56:10,275 DEBUG HandlerThread:3703 [handler.py:handle_request():146] handle_request: shutdown +2024-05-09 13:56:10,275 INFO HandlerThread:3703 [handler.py:finish():866] shutting down handler +2024-05-09 13:56:11,274 INFO WriterThread:3703 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_135532-o84u1qny/run-o84u1qny.wandb +2024-05-09 13:56:11,274 INFO SenderThread:3703 [sender.py:finish():1546] shutting down sender diff --git a/wandb/offline-run-20240509_135532-o84u1qny/logs/debug.log b/wandb/offline-run-20240509_135532-o84u1qny/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..26a686a38000e7dc79132e12f160d4a45ae58cf6 --- /dev/null +++ b/wandb/offline-run-20240509_135532-o84u1qny/logs/debug.log @@ -0,0 +1,27 @@ +2024-05-09 13:55:32,585 INFO MainThread:3486 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-09 13:55:32,586 INFO MainThread:3486 [wandb_setup.py:_flush():76] Configure stats pid to 3486 +2024-05-09 13:55:32,586 INFO MainThread:3486 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-09 13:55:32,586 INFO MainThread:3486 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-09 13:55:32,586 INFO MainThread:3486 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-09 13:55:32,586 INFO MainThread:3486 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-09 13:55:32,586 INFO MainThread:3486 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-09 13:55:32,586 INFO MainThread:3486 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-09 13:55:32,586 INFO MainThread:3486 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-09 13:55:32,587 INFO MainThread:3486 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_135532-o84u1qny/logs/debug.log +2024-05-09 13:55:32,587 INFO MainThread:3486 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_135532-o84u1qny/logs/debug-internal.log +2024-05-09 13:55:32,587 INFO MainThread:3486 [wandb_init.py:init():561] calling init triggers +2024-05-09 13:55:32,587 INFO MainThread:3486 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 32, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-09 13:55:32,587 INFO MainThread:3486 [wandb_init.py:init():611] starting backend +2024-05-09 13:55:32,587 INFO MainThread:3486 [wandb_init.py:init():615] setting up manager +2024-05-09 13:55:32,599 INFO MainThread:3486 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-09 13:55:32,601 INFO MainThread:3486 [wandb_init.py:init():623] backend started and connected +2024-05-09 13:55:32,608 INFO MainThread:3486 [wandb_init.py:init():715] updated telemetry +2024-05-09 13:55:32,623 INFO MainThread:3486 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-09 13:55:32,630 INFO MainThread:3486 [wandb_init.py:init():799] starting run threads in backend +2024-05-09 13:55:39,764 INFO MainThread:3486 [wandb_run.py:_console_start():2335] atexit reg +2024-05-09 13:55:39,764 INFO MainThread:3486 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-09 13:55:39,764 INFO MainThread:3486 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-09 13:55:39,764 INFO MainThread:3486 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-09 13:55:39,765 INFO MainThread:3486 [wandb_init.py:init():842] run started, returning control to user process +2024-05-09 13:56:11,277 WARNING MsgRouterThr:3486 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/offline-run-20240509_135532-o84u1qny/run-o84u1qny.wandb b/wandb/offline-run-20240509_135532-o84u1qny/run-o84u1qny.wandb new file mode 100644 index 0000000000000000000000000000000000000000..e8008db509c248930d9a6c1162031402fa368d1a Binary files /dev/null and b/wandb/offline-run-20240509_135532-o84u1qny/run-o84u1qny.wandb differ diff --git a/wandb/offline-run-20240509_140123-vqw25nkx/files/conda-environment.yaml b/wandb/offline-run-20240509_140123-vqw25nkx/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..fb5c46cd8a524560371e6601fcdbf9ef63efe33a --- /dev/null +++ b/wandb/offline-run-20240509_140123-vqw25nkx/files/conda-environment.yaml @@ -0,0 +1,668 @@ +name: base +channels: + - conda-forge + - defaults +dependencies: + - _anaconda_depends=2024.02=py311_mkl_1 + - _libgcc_mutex=0.1=conda_forge + - _openmp_mutex=4.5=2_gnu + - abseil-cpp=20211102.0=hd4dd3e8_0 + - aiobotocore=2.7.0=py311h06a4308_0 + - aiohttp=3.9.3=py311h5eee18b_0 + - aioitertools=0.7.1=pyhd3eb1b0_0 + - aiosignal=1.2.0=pyhd3eb1b0_0 + - alabaster=0.7.12=pyhd3eb1b0_0 + - altair=5.0.1=py311h06a4308_0 + - anaconda-anon-usage=0.4.3=py311hfc0e8ea_100 + - anaconda-catalogs=0.2.0=py311h06a4308_0 + - anaconda-client=1.12.3=py311h06a4308_0 + - anaconda-cloud-auth=0.1.4=py311h06a4308_0 + - anaconda-navigator=2.5.2=py311h06a4308_0 + - anaconda-project=0.11.1=py311h06a4308_0 + - anyio=4.2.0=py311h06a4308_0 + - aom=3.5.0=h27087fc_0 + - appdirs=1.4.4=pyhd3eb1b0_0 + - archspec=0.2.1=pyhd3eb1b0_0 + - argon2-cffi=21.3.0=pyhd3eb1b0_0 + - argon2-cffi-bindings=21.2.0=py311h5eee18b_0 + - arrow=1.2.3=py311h06a4308_1 + - arrow-cpp=14.0.2=h374c478_1 + - astroid=2.14.2=py311h06a4308_0 + - astropy=5.3.4=py311hf4808d0_0 + - asttokens=2.0.5=pyhd3eb1b0_0 + - async-lru=2.0.4=py311h06a4308_0 + - atomicwrites=1.4.0=py_0 + - attrs=23.1.0=py311h06a4308_0 + - automat=20.2.0=py_0 + - autopep8=1.6.0=pyhd3eb1b0_1 + - aws-c-auth=0.6.19=h5eee18b_0 + - aws-c-cal=0.5.20=hdbd6064_0 + - aws-c-common=0.8.5=h5eee18b_0 + - aws-c-compression=0.2.16=h5eee18b_0 + - aws-c-event-stream=0.2.15=h6a678d5_0 + - aws-c-http=0.6.25=h5eee18b_0 + - aws-c-io=0.13.10=h5eee18b_0 + - aws-c-mqtt=0.7.13=h5eee18b_0 + - aws-c-s3=0.1.51=hdbd6064_0 + - aws-c-sdkutils=0.1.6=h5eee18b_0 + - aws-checksums=0.1.13=h5eee18b_0 + - aws-crt-cpp=0.18.16=h6a678d5_0 + - aws-sdk-cpp=1.10.55=h721c034_0 + - babel=2.11.0=py311h06a4308_0 + - backports=1.1=pyhd3eb1b0_0 + - backports.functools_lru_cache=1.6.4=pyhd3eb1b0_0 + - backports.tempfile=1.0=pyhd3eb1b0_1 + - backports.weakref=1.0.post1=py_1 + - bcrypt=3.2.0=py311h5eee18b_1 + - beautifulsoup4=4.12.2=py311h06a4308_0 + - binaryornot=0.4.4=pyhd3eb1b0_1 + - black=23.11.0=py311h06a4308_0 + - blas=1.0=mkl + - bleach=4.1.0=pyhd3eb1b0_0 + - blinker=1.6.2=py311h06a4308_0 + - blosc=1.21.3=h6a678d5_0 + - bokeh=3.3.4=py311h92b7b1e_0 + - boltons=23.0.0=py311h06a4308_0 + - boost-cpp=1.82.0=hdb19cb5_2 + - botocore=1.31.64=py311h06a4308_0 + - bottleneck=1.3.7=py311hf4808d0_0 + - brotli=1.0.9=h5eee18b_7 + - brotli-bin=1.0.9=h5eee18b_7 + - brotli-python=1.0.9=py311h6a678d5_7 + - brunsli=0.1=h2531618_0 + - bzip2=1.0.8=h7b6447c_0 + - c-ares=1.19.1=h5eee18b_0 + - c-blosc2=2.12.0=h80c7b02_0 + - ca-certificates=2024.2.2=hbcca054_0 + - cachetools=4.2.2=pyhd3eb1b0_0 + - cairo=1.18.0=h3faef2a_0 + - certifi=2024.2.2=pyhd8ed1ab_0 + - cffi=1.16.0=py311h5eee18b_0 + - cfitsio=3.470=h5893167_7 + - chardet=4.0.0=py311h06a4308_1003 + - charls=2.2.0=h2531618_0 + - charset-normalizer=2.0.4=pyhd3eb1b0_0 + - click=8.1.7=py311h06a4308_0 + - cloudpickle=2.2.1=py311h06a4308_0 + - clyent=1.2.2=py311h06a4308_1 + - colorama=0.4.6=py311h06a4308_0 + - colorcet=3.0.1=py311h06a4308_0 + - comm=0.1.2=py311h06a4308_0 + - conda=24.1.2=py311h38be061_0 + - conda-build=24.1.2=py311h06a4308_0 + - conda-content-trust=0.2.0=py311h06a4308_0 + - conda-index=0.4.0=pyhd3eb1b0_0 + - conda-libmamba-solver=24.1.0=pyhd3eb1b0_0 + - conda-pack=0.6.0=pyhd3eb1b0_0 + - conda-package-handling=2.2.0=py311h06a4308_0 + - conda-package-streaming=0.9.0=py311h06a4308_0 + - conda-repo-cli=1.0.75=py311h06a4308_0 + - conda-token=0.4.0=pyhd3eb1b0_0 + - conda-verify=3.4.2=py_1 + - constantly=23.10.4=py311h06a4308_0 + - contourpy=1.2.0=py311hdb19cb5_0 + - cookiecutter=2.5.0=py311h06a4308_0 + - cryptography=42.0.2=py311hdda0065_0 + - cssselect=1.2.0=py311h06a4308_0 + - curl=8.5.0=hdbd6064_0 + - cycler=0.11.0=pyhd3eb1b0_0 + - cyrus-sasl=2.1.28=h52b45da_1 + - cytoolz=0.12.2=py311h5eee18b_0 + - dask=2023.11.0=py311h06a4308_0 + - dask-core=2023.11.0=py311h06a4308_0 + - datashader=0.16.0=py311h06a4308_0 + - dav1d=1.2.0=hd590300_0 + - dbus=1.13.18=hb2f20db_0 + - debugpy=1.6.7=py311h6a678d5_0 + - defusedxml=0.7.1=pyhd3eb1b0_0 + - diff-match-patch=20200713=pyhd3eb1b0_0 + - distributed=2023.11.0=py311h06a4308_0 + - distro=1.8.0=py311h06a4308_0 + - docstring-to-markdown=0.11=py311h06a4308_0 + - docutils=0.18.1=py311h06a4308_3 + - entrypoints=0.4=py311h06a4308_0 + - et_xmlfile=1.1.0=py311h06a4308_0 + - executing=0.8.3=pyhd3eb1b0_0 + - expat=2.5.0=h6a678d5_0 + - ffmpeg=6.0.0=gpl_h5bb71f3_100 + - filelock=3.13.1=py311h06a4308_0 + - flake8=6.0.0=py311h06a4308_0 + - flask=2.2.5=py311h06a4308_0 + - fmt=9.1.0=hdb19cb5_0 + - font-ttf-dejavu-sans-mono=2.37=hab24e00_0 + - font-ttf-inconsolata=3.000=h77eed37_0 + - font-ttf-source-code-pro=2.038=h77eed37_0 + - font-ttf-ubuntu=0.83=h77eed37_2 + - fontconfig=2.14.2=h14ed4e7_0 + - fonts-conda-ecosystem=1=0 + - fonts-conda-forge=1=0 + - fonttools=4.25.0=pyhd3eb1b0_0 + - freetype=2.12.1=h4a9f257_0 + - fribidi=1.0.10=h36c2ea0_0 + - frozenlist=1.4.0=py311h5eee18b_0 + - fsspec=2023.10.0=py311h06a4308_0 + - future=0.18.3=py311h06a4308_0 + - gensim=4.3.0=py311hba01205_1 + - gettext=0.22.5=h59595ed_2 + - gettext-tools=0.22.5=h59595ed_2 + - gflags=2.2.2=h6a678d5_1 + - giflib=5.2.1=h5eee18b_3 + - gitdb=4.0.7=pyhd3eb1b0_0 + - gitpython=3.1.37=py311h06a4308_0 + - glib=2.78.4=h6a678d5_0 + - glib-tools=2.78.4=h6a678d5_0 + - glog=0.5.0=h6a678d5_1 + - gmp=6.2.1=h295c915_3 + - gmpy2=2.1.2=py311hc9b5ff0_0 + - gnutls=3.7.9=hb077bed_0 + - graphite2=1.3.13=h59595ed_1003 + - greenlet=3.0.1=py311h6a678d5_0 + - grpc-cpp=1.48.2=he1ff14a_1 + - gst-plugins-base=1.14.1=h6a678d5_1 + - gstreamer=1.14.1=h5eee18b_1 + - h5py=3.9.0=py311hdd6beaf_0 + - harfbuzz=8.3.0=h3d44ed6_0 + - hdf5=1.12.1=h2b7332f_3 + - heapdict=1.0.1=pyhd3eb1b0_0 + - holoviews=1.18.3=py311h06a4308_0 + - hvplot=0.9.2=py311h06a4308_0 + - hyperlink=21.0.0=pyhd3eb1b0_0 + - icu=73.2=h59595ed_0 + - idna=3.4=py311h06a4308_0 + - imagecodecs=2023.1.23=py311h8105a5c_0 + - imageio=2.33.1=py311h06a4308_0 + - imagesize=1.4.1=py311h06a4308_0 + - imbalanced-learn=0.11.0=py311h06a4308_1 + - importlib-metadata=7.0.1=py311h06a4308_0 + - importlib_metadata=7.0.1=hd3eb1b0_0 + - incremental=22.10.0=pyhd3eb1b0_0 + - inflection=0.5.1=py311h06a4308_0 + - iniconfig=1.1.1=pyhd3eb1b0_0 + - intake=0.6.8=py311h06a4308_0 + - intel-openmp=2023.1.0=hdb19cb5_46306 + - intervaltree=3.1.0=pyhd3eb1b0_0 + - ipykernel=6.28.0=py311h06a4308_0 + - ipython=8.20.0=py311h06a4308_0 + - ipython_genutils=0.2.0=pyhd3eb1b0_1 + - ipywidgets=7.6.5=pyhd3eb1b0_2 + - isort=5.9.3=pyhd3eb1b0_0 + - itemadapter=0.3.0=pyhd3eb1b0_0 + - itemloaders=1.1.0=py311h06a4308_0 + - itsdangerous=2.0.1=pyhd3eb1b0_0 + - jaraco.classes=3.2.1=pyhd3eb1b0_0 + - jedi=0.18.1=py311h06a4308_1 + - jeepney=0.7.1=pyhd3eb1b0_0 + - jellyfish=1.0.1=py311hb02cf49_0 + - jinja2=3.1.3=py311h06a4308_0 + - jmespath=1.0.1=py311h06a4308_0 + - joblib=1.2.0=py311h06a4308_0 + - jpeg=9e=h5eee18b_1 + - jq=1.6=h27cfd23_1000 + - json5=0.9.6=pyhd3eb1b0_0 + - jsonpatch=1.32=pyhd3eb1b0_0 + - jsonpointer=2.1=pyhd3eb1b0_0 + - jsonschema=4.19.2=py311h06a4308_0 + - jsonschema-specifications=2023.7.1=py311h06a4308_0 + - jupyter=1.0.0=py311h06a4308_9 + - jupyter-lsp=2.2.0=py311h06a4308_0 + - jupyter_client=8.6.0=py311h06a4308_0 + - jupyter_console=6.6.3=py311h06a4308_0 + - jupyter_core=5.5.0=py311h06a4308_0 + - jupyter_events=0.8.0=py311h06a4308_0 + - jupyter_server=2.10.0=py311h06a4308_0 + - jupyter_server_terminals=0.4.4=py311h06a4308_1 + - jupyterlab=4.0.11=py311h06a4308_0 + - jupyterlab-variableinspector=3.1.0=py311h06a4308_0 + - jupyterlab_pygments=0.1.2=py_0 + - jupyterlab_server=2.25.1=py311h06a4308_0 + - jupyterlab_widgets=3.0.9=py311h06a4308_0 + - jxrlib=1.1=h7b6447c_2 + - keyring=23.13.1=py311h06a4308_0 + - kiwisolver=1.4.4=py311h6a678d5_0 + - krb5=1.20.1=h143b758_1 + - lame=3.100=h166bdaf_1003 + - lazy-object-proxy=1.6.0=py311h5eee18b_0 + - lazy_loader=0.3=py311h06a4308_0 + - lcms2=2.12=h3be6417_0 + - ld_impl_linux-64=2.38=h1181459_1 + - lerc=3.0=h295c915_0 + - libaec=1.0.4=he6710b0_1 + - libarchive=3.6.2=h6ac8c49_2 + - libasprintf=0.22.5=h661eb56_2 + - libasprintf-devel=0.22.5=h661eb56_2 + - libass=0.17.1=h8fe9dca_1 + - libavif=0.11.1=hff004cb_1 + - libboost=1.82.0=h109eef0_2 + - libbrotlicommon=1.0.9=h5eee18b_7 + - libbrotlidec=1.0.9=h5eee18b_7 + - libbrotlienc=1.0.9=h5eee18b_7 + - libclang=14.0.6=default_hc6dbbc7_1 + - libclang13=14.0.6=default_he11475f_1 + - libcups=2.4.2=h2d74bed_1 + - libcurl=8.5.0=h251f7ec_0 + - libdeflate=1.17=h5eee18b_1 + - libdrm=2.4.120=hd590300_0 + - libedit=3.1.20230828=h5eee18b_0 + - libev=4.33=h7f8727e_1 + - libevent=2.1.12=hdbd6064_1 + - libexpat=2.5.0=hcb278e6_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=13.2.0=h77fa898_7 + - libgettextpo=0.22.5=h59595ed_2 + - libgettextpo-devel=0.22.5=h59595ed_2 + - libgfortran-ng=11.2.0=h00389a5_1 + - libgfortran5=11.2.0=h1234567_1 + - libglib=2.78.4=hdc74915_0 + - libgomp=13.2.0=h77fa898_7 + - libiconv=1.16=h7f8727e_2 + - libidn2=2.3.7=hd590300_0 + - liblief=0.12.3=h6a678d5_0 + - libllvm14=14.0.6=hdb19cb5_3 + - libmamba=1.5.6=haf1ee3a_0 + - libmambapy=1.5.6=py311h2dafd23_0 + - libnghttp2=1.57.0=h2d74bed_0 + - libnsl=2.0.1=hd590300_0 + - libopus=1.3.1=h7f98852_1 + - libpciaccess=0.18=hd590300_0 + - libpng=1.6.39=h5eee18b_0 + - libpq=12.17=hdbd6064_0 + - libprotobuf=3.20.3=he621ea3_0 + - libsodium=1.0.18=h7b6447c_0 + - libsolv=0.7.24=he621ea3_0 + - libspatialindex=1.9.3=h2531618_0 + - libsqlite=3.45.3=h2797004_0 + - libssh2=1.10.0=hdbd6064_2 + - libstdcxx-ng=13.2.0=hc0a3c3a_7 + - libtasn1=4.19.0=h166bdaf_0 + - libthrift=0.15.0=h1795dd8_2 + - libtiff=4.5.1=h6a678d5_0 + - libunistring=0.9.10=h7f98852_0 + - libuuid=2.38.1=h0b41bf4_0 + - libva=2.21.0=hd590300_0 + - libvpx=1.13.1=h59595ed_0 + - libwebp-base=1.3.2=h5eee18b_0 + - libxcb=1.15=h7f8727e_0 + - libxcrypt=4.4.36=hd590300_1 + - libxkbcommon=1.0.1=h5eee18b_1 + - libxml2=2.10.4=hf1b16e4_1 + - libxslt=1.1.37=h5eee18b_1 + - libzlib=1.2.13=hd590300_5 + - libzopfli=1.0.3=he6710b0_0 + - linkify-it-py=2.0.0=py311h06a4308_0 + - llvmlite=0.42.0=py311h6a678d5_0 + - locket=1.0.0=py311h06a4308_0 + - lxml=4.9.3=py311hdbbb534_0 + - lz4=4.3.2=py311h5eee18b_0 + - lz4-c=1.9.4=h6a678d5_0 + - lzo=2.10=h7b6447c_2 + - markdown=3.4.1=py311h06a4308_0 + - markdown-it-py=2.2.0=py311h06a4308_1 + - markupsafe=2.1.3=py311h5eee18b_0 + - matplotlib=3.8.0=py311h06a4308_0 + - matplotlib-base=3.8.0=py311ha02d727_0 + - matplotlib-inline=0.1.6=py311h06a4308_0 + - mccabe=0.7.0=pyhd3eb1b0_0 + - mdit-py-plugins=0.3.0=py311h06a4308_0 + - mdurl=0.1.0=py311h06a4308_0 + - menuinst=2.0.2=py311h06a4308_0 + - mistune=2.0.4=py311h06a4308_0 + - mkl=2023.1.0=h213fc3f_46344 + - mkl-service=2.4.0=py311h5eee18b_1 + - mkl_fft=1.3.8=py311h5eee18b_0 + - mkl_random=1.2.4=py311hdb19cb5_0 + - more-itertools=10.1.0=py311h06a4308_0 + - mpc=1.1.0=h10f8cd9_1 + - mpfr=4.0.2=hb69a4c5_1 + - mpmath=1.3.0=py311h06a4308_0 + - msgpack-python=1.0.3=py311hdb19cb5_0 + - multidict=6.0.4=py311h5eee18b_0 + - multipledispatch=0.6.0=py311h06a4308_0 + - munkres=1.1.4=py_0 + - mypy=1.8.0=py311h5eee18b_0 + - mypy_extensions=1.0.0=py311h06a4308_0 + - mysql=5.7.24=h721c034_2 + - navigator-updater=0.4.0=py311h06a4308_1 + - nbclient=0.8.0=py311h06a4308_0 + - nbconvert=7.10.0=py311h06a4308_0 + - nbformat=5.9.2=py311h06a4308_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.6.0=py311h06a4308_0 + - nettle=3.9.1=h7ab15ed_0 + - networkx=3.1=py311h06a4308_0 + - nltk=3.8.1=py311h06a4308_0 + - notebook=7.0.8=py311h06a4308_0 + - notebook-shim=0.2.3=py311h06a4308_0 + - nspr=4.35=h6a678d5_0 + - nss=3.89.1=h6a678d5_0 + - numba=0.59.1=py311h96b013e_0 + - numexpr=2.8.7=py311h65dcdc2_0 + - numpy=1.26.4=py311h08b1b3b_0 + - numpy-base=1.26.4=py311hf175353_0 + - numpydoc=1.5.0=py311h06a4308_0 + - oniguruma=6.9.7.1=h27cfd23_0 + - openh264=2.3.1=hcb278e6_2 + - openjpeg=2.4.0=h3ad879b_0 + - openpyxl=3.0.10=py311h5eee18b_0 + - openssl=3.3.0=hd590300_0 + - orc=1.7.4=hb3bc3d3_1 + - overrides=7.4.0=py311h06a4308_0 + - p11-kit=0.24.1=hc5aa10d_0 + - packaging=23.1=py311h06a4308_0 + - pandas=2.1.4=py311ha02d727_0 + - pandocfilters=1.5.0=pyhd3eb1b0_0 + - panel=1.3.8=py311h06a4308_0 + - param=2.0.2=py311h06a4308_0 + - parsel=1.8.1=py311h06a4308_0 + - parso=0.8.3=pyhd3eb1b0_0 + - partd=1.4.1=py311h06a4308_0 + - patch=2.7.6=h7b6447c_1001 + - patchelf=0.17.2=h6a678d5_0 + - pathlib=1.0.1=pyhd3eb1b0_1 + - pathspec=0.10.3=py311h06a4308_0 + - patsy=0.5.3=py311h06a4308_0 + - pcre2=10.42=hebb0a14_0 + - pexpect=4.8.0=pyhd3eb1b0_3 + - pickleshare=0.7.5=pyhd3eb1b0_1003 + - pixman=0.43.2=h59595ed_0 + - pkce=1.0.3=py311h06a4308_0 + - pkginfo=1.9.6=py311h06a4308_0 + - platformdirs=3.10.0=py311h06a4308_0 + - plotly=5.9.0=py311h06a4308_0 + - pluggy=1.0.0=py311h06a4308_1 + - ply=3.11=py311h06a4308_0 + - prometheus_client=0.14.1=py311h06a4308_0 + - prompt-toolkit=3.0.43=py311h06a4308_0 + - prompt_toolkit=3.0.43=hd3eb1b0_0 + - protego=0.1.16=py_0 + - protobuf=3.20.3=py311h6a678d5_0 + - psutil=5.9.0=py311h5eee18b_0 + - ptyprocess=0.7.0=pyhd3eb1b0_2 + - pure_eval=0.2.2=pyhd3eb1b0_0 + - py-cpuinfo=9.0.0=py311h06a4308_0 + - py-lief=0.12.3=py311h6a678d5_0 + - pyarrow=14.0.2=py311hb6e97c4_0 + - pyasn1=0.4.8=pyhd3eb1b0_0 + - pyasn1-modules=0.2.8=py_0 + - pybind11-abi=4=hd3eb1b0_1 + - pycodestyle=2.10.0=py311h06a4308_0 + - pycosat=0.6.6=py311h5eee18b_0 + - pycparser=2.21=pyhd3eb1b0_0 + - pyct=0.5.0=py311h06a4308_0 + - pycurl=7.45.2=py311hdbd6064_1 + - pydeck=0.8.0=py311h06a4308_2 + - pydispatcher=2.0.5=py311h06a4308_2 + - pydocstyle=6.3.0=py311h06a4308_0 + - pyerfa=2.0.0=py311h5eee18b_0 + - pyflakes=3.0.1=py311h06a4308_0 + - pygments=2.15.1=py311h06a4308_1 + - pyjwt=2.4.0=py311h06a4308_0 + - pylint=2.16.2=py311h06a4308_0 + - pylint-venv=2.3.0=py311h06a4308_0 + - pyls-spyder=0.4.0=pyhd3eb1b0_0 + - pyodbc=5.0.1=py311h6a678d5_0 + - pyopenssl=24.0.0=py311h06a4308_0 + - pyparsing=3.0.9=py311h06a4308_0 + - pyqt=5.15.10=py311h6a678d5_0 + - pyqt5-sip=12.13.0=py311h5eee18b_0 + - pyqtwebengine=5.15.10=py311h6a678d5_0 + - pysocks=1.7.1=py311h06a4308_0 + - pytables=3.9.2=py311h9d13977_0 + - pytest=7.4.0=py311h06a4308_0 + - python=3.11.8=hab00c5b_0_cpython + - python-dateutil=2.8.2=pyhd3eb1b0_0 + - python-dotenv=0.21.0=py311h06a4308_0 + - python-fastjsonschema=2.16.2=py311h06a4308_0 + - python-json-logger=2.0.7=py311h06a4308_0 + - python-libarchive-c=2.9=pyhd3eb1b0_1 + - python-lmdb=1.4.1=py311h6a678d5_0 + - python-lsp-black=1.2.1=py311h06a4308_0 + - python-lsp-jsonrpc=1.0.0=pyhd3eb1b0_0 + - python-lsp-server=1.7.2=py311h06a4308_0 + - python-slugify=5.0.2=pyhd3eb1b0_0 + - python-snappy=0.6.1=py311h6a678d5_0 + - python-tzdata=2023.3=pyhd3eb1b0_0 + - python_abi=3.11=4_cp311 + - pytoolconfig=1.2.6=py311h06a4308_0 + - pytz=2023.3.post1=py311h06a4308_0 + - pyviz_comms=3.0.0=py311h06a4308_0 + - pywavelets=1.5.0=py311hf4808d0_0 + - pyxdg=0.27=pyhd3eb1b0_0 + - pyyaml=6.0.1=py311h5eee18b_0 + - pyzmq=25.1.2=py311h6a678d5_0 + - qdarkstyle=3.0.2=pyhd3eb1b0_0 + - qstylizer=0.2.2=py311h06a4308_0 + - qt-main=5.15.2=h53bd1ea_10 + - qt-webengine=5.15.9=h9ab4d14_7 + - qtawesome=1.2.2=py311h06a4308_0 + - qtconsole=5.4.2=py311h06a4308_0 + - qtpy=2.4.1=py311h06a4308_0 + - queuelib=1.6.2=py311h06a4308_0 + - re2=2022.04.01=h295c915_0 + - readline=8.2=h5eee18b_0 + - referencing=0.30.2=py311h06a4308_0 + - regex=2023.10.3=py311h5eee18b_0 + - reproc=14.2.4=h295c915_1 + - reproc-cpp=14.2.4=h295c915_1 + - requests=2.31.0=py311h06a4308_1 + - requests-file=1.5.1=pyhd3eb1b0_0 + - requests-toolbelt=1.0.0=py311h06a4308_0 + - rfc3339-validator=0.1.4=py311h06a4308_0 + - rfc3986-validator=0.1.1=py311h06a4308_0 + - rich=13.3.5=py311h06a4308_0 + - rope=1.7.0=py311h06a4308_0 + - rpds-py=0.10.6=py311hb02cf49_0 + - rtree=1.0.1=py311h06a4308_0 + - ruamel.yaml=0.17.21=py311h5eee18b_0 + - ruamel_yaml=0.17.21=py311h5eee18b_0 + - s2n=1.3.27=hdbd6064_0 + - s3fs=2023.10.0=py311h06a4308_0 + - scikit-image=0.22.0=py311ha02d727_0 + - scikit-learn=1.2.2=py311h6a678d5_1 + - scipy=1.11.4=py311h08b1b3b_0 + - scrapy=2.8.0=py311h06a4308_0 + - seaborn=0.12.2=py311h06a4308_0 + - secretstorage=3.3.1=py311h06a4308_1 + - semver=2.13.0=pyhd3eb1b0_0 + - send2trash=1.8.2=py311h06a4308_0 + - service_identity=18.1.0=pyhd3eb1b0_1 + - setuptools=68.2.2=py311h06a4308_0 + - sip=6.7.12=py311h6a678d5_0 + - six=1.16.0=pyhd3eb1b0_1 + - smart_open=5.2.1=py311h06a4308_0 + - smmap=4.0.0=pyhd3eb1b0_0 + - snappy=1.1.10=h6a678d5_1 + - sniffio=1.3.0=py311h06a4308_0 + - snowballstemmer=2.2.0=pyhd3eb1b0_0 + - sortedcontainers=2.4.0=pyhd3eb1b0_0 + - soupsieve=2.5=py311h06a4308_0 + - sphinx=5.0.2=py311h06a4308_0 + - sphinxcontrib-applehelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-devhelp=1.0.2=pyhd3eb1b0_0 + - sphinxcontrib-htmlhelp=2.0.0=pyhd3eb1b0_0 + - sphinxcontrib-jsmath=1.0.1=pyhd3eb1b0_0 + - sphinxcontrib-qthelp=1.0.3=pyhd3eb1b0_0 + - sphinxcontrib-serializinghtml=1.1.5=pyhd3eb1b0_0 + - spyder=5.4.3=py311h06a4308_1 + - spyder-kernels=2.4.4=py311h06a4308_0 + - sqlalchemy=2.0.25=py311h5eee18b_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.2.0=pyhd3eb1b0_0 + - statsmodels=0.14.0=py311hf4808d0_0 + - streamlit=1.30.0=py311h06a4308_0 + - svt-av1=1.5.0=h59595ed_0 + - sympy=1.12=py311h06a4308_0 + - tabulate=0.9.0=py311h06a4308_0 + - tbb=2021.8.0=hdb19cb5_0 + - tblib=1.7.0=pyhd3eb1b0_0 + - tenacity=8.2.2=py311h06a4308_0 + - terminado=0.17.1=py311h06a4308_0 + - text-unidecode=1.3=pyhd3eb1b0_0 + - textdistance=4.2.1=pyhd3eb1b0_0 + - threadpoolctl=2.2.0=pyh0d69192_0 + - three-merge=0.1.1=pyhd3eb1b0_0 + - tifffile=2023.4.12=py311h06a4308_0 + - tinycss2=1.2.1=py311h06a4308_0 + - tk=8.6.13=noxft_h4845f30_101 + - tldextract=3.2.0=pyhd3eb1b0_0 + - toml=0.10.2=pyhd3eb1b0_0 + - toolz=0.12.0=py311h06a4308_0 + - tornado=6.3.3=py311h5eee18b_0 + - tqdm=4.65.0=py311h92b7b1e_0 + - traitlets=5.7.1=py311h06a4308_0 + - truststore=0.8.0=py311h06a4308_0 + - twisted=23.10.0=py311h06a4308_0 + - typing-extensions=4.9.0=py311h06a4308_1 + - typing_extensions=4.9.0=py311h06a4308_1 + - tzdata=2023d=h04d1e81_0 + - tzlocal=2.1=py311h06a4308_1 + - uc-micro-py=1.0.1=py311h06a4308_0 + - ujson=5.4.0=py311h6a678d5_0 + - unidecode=1.2.0=pyhd3eb1b0_0 + - unixodbc=2.3.11=h5eee18b_0 + - urllib3=2.0.7=py311h06a4308_0 + - utf8proc=2.6.1=h5eee18b_1 + - validators=0.18.2=pyhd3eb1b0_0 + - w3lib=2.1.2=py311h06a4308_0 + - watchdog=2.1.6=py311h06a4308_0 + - wcwidth=0.2.5=pyhd3eb1b0_0 + - webencodings=0.5.1=py311h06a4308_1 + - websocket-client=0.58.0=py311h06a4308_4 + - werkzeug=2.2.3=py311h06a4308_0 + - whatthepatch=1.0.2=py311h06a4308_0 + - wheel=0.41.2=py311h06a4308_0 + - widgetsnbextension=3.5.2=py311h06a4308_1 + - wrapt=1.14.1=py311h5eee18b_0 + - wurlitzer=3.0.2=py311h06a4308_0 + - x264=1!164.3095=h166bdaf_2 + - x265=3.5=h924138e_3 + - xarray=2023.6.0=py311h06a4308_0 + - xorg-fixesproto=5.0=h7f98852_1002 + - xorg-kbproto=1.0.7=h7f98852_1002 + - xorg-libice=1.1.1=hd590300_0 + - xorg-libsm=1.2.4=h7391055_0 + - xorg-libx11=1.8.9=h8ee46fc_0 + - xorg-libxext=1.3.4=h0b41bf4_2 + - xorg-libxfixes=5.0.3=h7f98852_1004 + - xorg-libxrender=0.9.11=hd590300_0 + - xorg-renderproto=0.11.1=h7f98852_1002 + - xorg-xextproto=7.3.0=h0b41bf4_1003 + - xorg-xproto=7.0.31=h7f98852_1007 + - xyzservices=2022.9.0=py311h06a4308_1 + - xz=5.4.5=h5eee18b_0 + - yaml=0.2.5=h7b6447c_0 + - yaml-cpp=0.8.0=h6a678d5_0 + - yapf=0.31.0=pyhd3eb1b0_0 + - yarl=1.9.3=py311h5eee18b_0 + - zeromq=4.3.5=h6a678d5_0 + - zfp=1.0.0=h6a678d5_0 + - zict=3.0.0=py311h06a4308_0 + - zipp=3.17.0=py311h06a4308_0 + - zlib=1.2.13=hd590300_5 + - zlib-ng=2.0.7=h5eee18b_0 + - zope=1.0=py311h06a4308_1 + - zope.interface=5.4.0=py311h5eee18b_0 + - zstandard=0.19.0=py311h5eee18b_0 + - zstd=1.5.5=hc292b87_0 + - pip: + - accelerate==0.30.0 + - aiofiles==23.2.1 + - annotated-types==0.6.0 + - bitsandbytes==0.41.0 + - cmake==3.29.2 + - datasets==2.19.0 + - decorator==4.4.2 + - deepspeed==0.12.6 + - dill==0.3.8 + - docker-pycreds==0.4.0 + - docstring-parser==0.16 + - draccus==0.7.1 + - einops==0.6.1 + - einops-exts==0.0.4 + - fastapi==0.110.2 + - ffmpy==0.3.2 + - flash-attn==2.5.8 + - gradio==4.16.0 + - gradio-client==0.8.1 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.3 + - httpx==0.24.0 + - huggingface-hub==0.23.0 + - icecream==2.1.3 + - imageio-ffmpeg==0.4.9 + - importlib-resources==6.4.0 + - jsonlines==4.0.0 + - lit==18.1.4 + - llava==1.2.2.post1 + - markdown2==2.4.13 + - mergedeep==1.3.4 + - moviepy==1.0.3 + - multiprocess==0.70.16 + - ninja==1.11.1.1 + - nvidia-cublas-cu11==11.10.3.66 + - nvidia-cublas-cu12==12.1.3.1 + - nvidia-cuda-cupti-cu11==11.7.101 + - nvidia-cuda-cupti-cu12==12.1.105 + - nvidia-cuda-nvrtc-cu11==11.7.99 + - nvidia-cuda-nvrtc-cu12==12.1.105 + - nvidia-cuda-runtime-cu11==11.7.99 + - nvidia-cuda-runtime-cu12==12.1.105 + - nvidia-cudnn-cu11==8.5.0.96 + - nvidia-cudnn-cu12==8.9.2.26 + - nvidia-cufft-cu11==10.9.0.58 + - nvidia-cufft-cu12==11.0.2.54 + - nvidia-curand-cu11==10.2.10.91 + - nvidia-curand-cu12==10.3.2.106 + - nvidia-cusolver-cu11==11.4.0.1 + - nvidia-cusolver-cu12==11.4.5.107 + - nvidia-cusparse-cu11==11.7.4.91 + - nvidia-cusparse-cu12==12.1.0.106 + - nvidia-nccl-cu11==2.14.3 + - nvidia-nccl-cu12==2.18.1 + - nvidia-nvjitlink-cu12==12.4.127 + - nvidia-nvtx-cu11==11.7.91 + - nvidia-nvtx-cu12==12.1.105 + - opencv-python==4.9.0.80 + - opencv-python-headless==4.9.0.80 + - orjson==3.10.1 + - peft==0.10.0 + - pillow==10.3.0 + - pip==24.0 + - prismatic==0.0.1 + - proglog==0.1.10 + - pyarrow-hotfix==0.6 + - pydantic==2.7.1 + - pydantic-core==2.18.2 + - pydub==0.25.1 + - pynvml==11.5.0 + - python-multipart==0.0.9 + - pyyaml-include==1.4.1 + - ruff==0.4.2 + - safetensors==0.4.3 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==2.0.1 + - setproctitle==1.3.3 + - shellingham==1.5.4 + - shortuuid==1.0.13 + - shtab==1.7.1 + - starlette==0.37.2 + - svgwrite==1.4.3 + - timm==0.9.16 + - tokenizers==0.14.1 + - tomlkit==0.12.0 + - torch==2.1.2 + - torchaudio==2.1.2 + - torchvision==0.16.2 + - transformers==4.40.1 + - triton==2.1.0 + - trl==0.8.6 + - typer==0.12.3 + - typing-inspect==0.9.0 + - tyro==0.8.3 + - uvicorn==0.29.0 + - wandb==0.16.6 + - wavedrom==2.0.3.post3 + - websockets==11.0.3 + - xxhash==3.4.1 +prefix: /scratch/groups/willhies/sschmidg/Anaconda3 diff --git a/wandb/offline-run-20240509_140123-vqw25nkx/files/wandb-metadata.json b/wandb/offline-run-20240509_140123-vqw25nkx/files/wandb-metadata.json new file mode 100644 index 0000000000000000000000000000000000000000..b5a769117a47be551e84c8e4dcba4e1b21bb1051 --- /dev/null +++ b/wandb/offline-run-20240509_140123-vqw25nkx/files/wandb-metadata.json @@ -0,0 +1,399 @@ +{ + "os": "Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17", + "python": "3.11.8", + "heartbeatAt": "2024-05-09T21:01:23.197793", + "startedAt": "2024-05-09T21:01:23.056501", + "docker": null, + "cuda": null, + "args": [ + "--model.type", + "one-stage+7b", + "--model.model_id", + "testmodel", + "--model.vision_backbone_id", + "clip-vit-b", + "--model.image_resize_strategy", + "letterbox", + "--model.llm_backbone_id", + "llama2-7b-chat", + "--model.finetune_global_batch_size", + "128", + "--model.finetune_per_device_batch_size", + "16", + "--model.finetune_epochs", + "1", + "--dataset.type", + "med-instruct" + ], + "state": "running", + "program": "/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py", + "codePathLocal": "scripts/pretrain.py", + "codePath": "scripts/pretrain.py", + "git": { + "remote": "https://github.com/TRI-ML/prismatic-vlms", + "commit": "8ddeda915e861a7f4cbe7ce92f1407f4feb6a181" + }, + "email": null, + "root": "/scratch/groups/willhies/sschmidg/prismatic-vlms", + "host": "sh03-11n01.int", + "username": "sschmidg", + "executable": "/scratch/groups/willhies/sschmidg/Anaconda3/bin/python", + "cpu_count": 64, + "cpu_count_logical": 64, + "cpu_freq": { + "current": 1.9468749999999992, + "min": 1500.0, + "max": 2800.0 + }, + "cpu_freq_per_core": [ + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.1, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 2.8, + "min": 1500.0, + "max": 2800.0 + }, + { + "current": 1.5, + "min": 1500.0, + "max": 2800.0 + } + ], + "disk": { + "/": { + "total": 23.98828125, + "used": 5.626476287841797 + } + }, + "gpu": "NVIDIA A100-SXM4-80GB", + "gpu_count": 4, + "gpu_devices": [ + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + }, + { + "name": "NVIDIA A100-SXM4-80GB", + "memory_total": 85899345920 + } + ], + "memory": { + "total": 503.5838279724121 + } +} diff --git a/wandb/offline-run-20240509_140123-vqw25nkx/files/wandb-summary.json b/wandb/offline-run-20240509_140123-vqw25nkx/files/wandb-summary.json new file mode 100644 index 0000000000000000000000000000000000000000..c705b16012d5dda581b1f3ccbe2518fd9c91ae53 --- /dev/null +++ b/wandb/offline-run-20240509_140123-vqw25nkx/files/wandb-summary.json @@ -0,0 +1 @@ +{"_wandb": {"runtime": 452}} \ No newline at end of file diff --git a/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug-internal.log b/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..de9b3fe2a2a3fb9dfd5c9b35160940acf01b9258 --- /dev/null +++ b/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug-internal.log @@ -0,0 +1,337 @@ +2024-05-09 14:01:23,076 INFO StreamThr :4861 [internal.py:wandb_internal():86] W&B internal server running at pid: 4861, started at: 2024-05-09 14:01:23.074424 +2024-05-09 14:01:23,078 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status +2024-05-09 14:01:23,086 INFO WriterThread:4861 [datastore.py:open_for_write():87] open: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/run-vqw25nkx.wandb +2024-05-09 14:01:23,143 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: run_start +2024-05-09 14:01:23,165 DEBUG HandlerThread:4861 [system_info.py:__init__():26] System info init +2024-05-09 14:01:23,165 DEBUG HandlerThread:4861 [system_info.py:__init__():41] System info init done +2024-05-09 14:01:23,165 INFO HandlerThread:4861 [system_monitor.py:start():194] Starting system monitor +2024-05-09 14:01:23,165 INFO SystemMonitor:4861 [system_monitor.py:_start():158] Starting system asset monitoring threads +2024-05-09 14:01:23,165 INFO HandlerThread:4861 [system_monitor.py:probe():214] Collecting system info +2024-05-09 14:01:23,167 INFO SystemMonitor:4861 [interfaces.py:start():190] Started cpu monitoring +2024-05-09 14:01:23,168 INFO SystemMonitor:4861 [interfaces.py:start():190] Started disk monitoring +2024-05-09 14:01:23,169 INFO SystemMonitor:4861 [interfaces.py:start():190] Started gpu monitoring +2024-05-09 14:01:23,170 INFO SystemMonitor:4861 [interfaces.py:start():190] Started memory monitoring +2024-05-09 14:01:23,170 INFO SystemMonitor:4861 [interfaces.py:start():190] Started network monitoring +2024-05-09 14:01:23,197 DEBUG HandlerThread:4861 [system_info.py:probe():150] Probing system +2024-05-09 14:01:23,199 DEBUG HandlerThread:4861 [system_info.py:_probe_git():135] Probing git +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_info.py:_probe_git():143] Probing git done +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_info.py:probe():198] Probing system done +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_monitor.py:probe():223] {'os': 'Linux-3.10.0-1160.114.2.el7.x86_64-x86_64-with-glibc2.17', 'python': '3.11.8', 'heartbeatAt': '2024-05-09T21:01:23.197793', 'startedAt': '2024-05-09T21:01:23.056501', 'docker': None, 'cuda': None, 'args': ('--model.type', 'one-stage+7b', '--model.model_id', 'testmodel', '--model.vision_backbone_id', 'clip-vit-b', '--model.image_resize_strategy', 'letterbox', '--model.llm_backbone_id', 'llama2-7b-chat', '--model.finetune_global_batch_size', '128', '--model.finetune_per_device_batch_size', '16', '--model.finetune_epochs', '1', '--dataset.type', 'med-instruct'), 'state': 'running', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'codePathLocal': 'scripts/pretrain.py', 'codePath': 'scripts/pretrain.py', 'git': {'remote': 'https://github.com/TRI-ML/prismatic-vlms', 'commit': '8ddeda915e861a7f4cbe7ce92f1407f4feb6a181'}, 'email': None, 'root': '/scratch/groups/willhies/sschmidg/prismatic-vlms', 'host': 'sh03-11n01.int', 'username': 'sschmidg', 'executable': '/scratch/groups/willhies/sschmidg/Anaconda3/bin/python', 'cpu_count': 64, 'cpu_count_logical': 64, 'cpu_freq': {'current': 1.9468749999999992, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.1, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 1.5, 'min': 1500.0, 'max': 2800.0}], 'disk': {'/': {'total': 23.98828125, 'used': 5.626476287841797}}, 'gpu': 'NVIDIA A100-SXM4-80GB', 'gpu_count': 4, 'gpu_devices': [{'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}, {'name': 'NVIDIA A100-SXM4-80GB', 'memory_total': 85899345920}], 'memory': {'total': 503.5838279724121}} +2024-05-09 14:01:23,209 INFO HandlerThread:4861 [system_monitor.py:probe():224] Finished collecting system info +2024-05-09 14:01:23,209 INFO HandlerThread:4861 [system_monitor.py:probe():227] Publishing system info +2024-05-09 14:01:23,209 DEBUG HandlerThread:4861 [system_info.py:_save_conda():207] Saving list of conda packages installed into the current environment +2024-05-09 14:01:30,038 DEBUG HandlerThread:4861 [system_info.py:_save_conda():222] Saving conda packages done +2024-05-09 14:01:30,042 INFO HandlerThread:4861 [system_monitor.py:probe():229] Finished publishing system info +2024-05-09 14:01:30,047 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:30,048 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: keepalive +2024-05-09 14:01:30,048 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:30,270 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: python_packages +2024-05-09 14:01:34,049 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:34,050 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:39,051 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:39,051 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:44,052 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:44,052 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:49,053 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:49,054 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:51,803 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:01:54,055 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:54,055 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:01:59,056 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:01:59,056 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:04,057 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:04,057 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:09,058 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:09,059 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:10,625 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:02:14,060 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:14,060 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:19,061 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:19,061 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:23,171 DEBUG SystemMonitor:4861 [system_monitor.py:_start():172] Starting system metrics aggregation loop +2024-05-09 14:02:24,062 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:24,062 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:28,604 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:02:29,063 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:29,063 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:34,064 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:34,065 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:39,065 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:39,066 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:44,066 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:44,067 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:45,970 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:02:49,068 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:49,068 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:54,069 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:54,069 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:02:59,070 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:02:59,071 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:03,109 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:04,072 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:04,072 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:09,073 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:09,073 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:14,074 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:14,075 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:19,075 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:19,076 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:19,636 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:24,077 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:24,077 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:29,078 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:29,078 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:34,079 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:34,079 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:36,299 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:39,080 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:39,081 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:44,082 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:44,082 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:49,083 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:49,083 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:54,084 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:54,085 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:03:54,227 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:03:59,085 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:03:59,086 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:04,087 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:04,087 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:09,088 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:09,088 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:10,990 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:04:14,089 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:14,089 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:19,090 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:19,091 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:24,092 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:24,092 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:27,771 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:04:29,093 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:29,094 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:34,095 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:34,095 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:39,096 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:39,096 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:44,097 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:44,098 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:44,534 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:04:49,098 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:49,099 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:54,100 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:54,100 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:04:59,101 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:04:59,101 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:01,175 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:04,102 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:04,103 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:09,103 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:09,104 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:14,105 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:14,105 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:17,930 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:19,106 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:19,106 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:24,107 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:24,107 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:29,108 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:29,108 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:34,109 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:34,110 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:36,171 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:39,111 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:39,111 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:44,112 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:44,112 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:49,113 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:49,113 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:52,984 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:05:54,114 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:54,115 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:05:59,115 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:05:59,116 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:04,117 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:04,117 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:09,118 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:09,118 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:11,350 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:06:14,119 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:14,119 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:19,120 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:19,120 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:24,121 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:24,122 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:29,068 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:06:29,122 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:29,123 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:34,123 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:34,124 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:39,125 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:39,125 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:44,126 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:44,126 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:46,015 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:06:49,127 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:49,128 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:54,129 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:54,129 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:06:59,130 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:06:59,130 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:02,938 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:04,131 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:04,131 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:09,132 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:09,132 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:14,133 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:14,134 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:19,134 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:19,135 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:19,747 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:24,135 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:24,136 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:29,137 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:29,137 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:34,138 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:34,138 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:36,532 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:39,139 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:39,140 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:44,141 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:44,141 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:49,142 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:49,142 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:52,968 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:07:54,143 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:54,144 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:07:59,144 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:07:59,145 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:04,146 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:04,146 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:09,147 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:09,147 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:09,692 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:08:14,148 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:14,149 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:19,149 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:19,150 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:24,150 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:24,151 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:28,467 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:08:29,152 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:29,152 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:34,153 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:34,153 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:39,154 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:39,154 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:44,155 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:44,156 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:45,278 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: partial_history +2024-05-09 14:08:49,157 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:49,157 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:54,158 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:54,159 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:55,939 DEBUG SenderThread:4861 [sender.py:send():379] send: exit +2024-05-09 14:08:55,939 INFO SenderThread:4861 [sender.py:send_exit():586] handling exit code: 255 +2024-05-09 14:08:55,939 INFO SenderThread:4861 [sender.py:send_exit():588] handling runtime: 452 +2024-05-09 14:08:55,943 INFO SenderThread:4861 [sender.py:_save_file():1390] saving file wandb-summary.json with policy end +2024-05-09 14:08:55,943 INFO SenderThread:4861 [sender.py:send_exit():594] send defer +2024-05-09 14:08:55,944 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,944 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 0 +2024-05-09 14:08:55,944 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,944 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 0 +2024-05-09 14:08:55,944 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 1 +2024-05-09 14:08:55,944 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,944 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 1 +2024-05-09 14:08:55,945 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,945 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 1 +2024-05-09 14:08:55,945 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 2 +2024-05-09 14:08:55,945 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,945 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 2 +2024-05-09 14:08:55,945 INFO HandlerThread:4861 [system_monitor.py:finish():203] Stopping system monitor +2024-05-09 14:08:55,946 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined cpu monitor +2024-05-09 14:08:55,946 DEBUG SystemMonitor:4861 [system_monitor.py:_start():179] Finished system metrics aggregation loop +2024-05-09 14:08:55,946 DEBUG SystemMonitor:4861 [system_monitor.py:_start():183] Publishing last batch of metrics +2024-05-09 14:08:55,954 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined disk monitor +2024-05-09 14:08:55,971 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined gpu monitor +2024-05-09 14:08:55,971 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined memory monitor +2024-05-09 14:08:55,971 INFO HandlerThread:4861 [interfaces.py:finish():202] Joined network monitor +2024-05-09 14:08:55,972 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,972 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 2 +2024-05-09 14:08:55,972 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 3 +2024-05-09 14:08:55,972 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,972 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 3 +2024-05-09 14:08:55,973 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,973 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 3 +2024-05-09 14:08:55,973 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 4 +2024-05-09 14:08:55,973 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,973 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 4 +2024-05-09 14:08:55,974 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 4 +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 5 +2024-05-09 14:08:55,974 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,974 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 5 +2024-05-09 14:08:55,974 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 5 +2024-05-09 14:08:55,974 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 6 +2024-05-09 14:08:55,974 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,975 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 6 +2024-05-09 14:08:55,975 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 6 +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 7 +2024-05-09 14:08:55,975 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: status_report +2024-05-09 14:08:55,975 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,975 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 7 +2024-05-09 14:08:55,975 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: status_report +2024-05-09 14:08:55,975 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 7 +2024-05-09 14:08:55,975 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 8 +2024-05-09 14:08:55,975 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 8 +2024-05-09 14:08:55,976 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 8 +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 9 +2024-05-09 14:08:55,976 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 9 +2024-05-09 14:08:55,976 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 9 +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 10 +2024-05-09 14:08:55,976 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 10 +2024-05-09 14:08:55,976 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 10 +2024-05-09 14:08:55,976 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 11 +2024-05-09 14:08:55,976 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,976 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 11 +2024-05-09 14:08:55,977 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 11 +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 12 +2024-05-09 14:08:55,977 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,977 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 12 +2024-05-09 14:08:55,977 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 12 +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 13 +2024-05-09 14:08:55,977 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,977 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 13 +2024-05-09 14:08:55,977 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 13 +2024-05-09 14:08:55,977 INFO SenderThread:4861 [sender.py:transition_state():614] send defer: 14 +2024-05-09 14:08:55,977 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: defer +2024-05-09 14:08:55,978 DEBUG SenderThread:4861 [sender.py:send():379] send: final +2024-05-09 14:08:55,978 INFO HandlerThread:4861 [handler.py:handle_request_defer():172] handle defer: 14 +2024-05-09 14:08:55,978 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: defer +2024-05-09 14:08:55,978 INFO SenderThread:4861 [sender.py:send_request_defer():610] handle sender defer: 14 +2024-05-09 14:08:55,978 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-09 14:08:55,979 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: poll_exit +2024-05-09 14:08:55,979 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: poll_exit +2024-05-09 14:08:55,979 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: poll_exit +2024-05-09 14:08:55,979 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: server_info +2024-05-09 14:08:55,980 DEBUG SenderThread:4861 [sender.py:send_request():406] send_request: server_info +2024-05-09 14:08:55,980 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: get_summary +2024-05-09 14:08:55,980 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: sampled_history +2024-05-09 14:08:55,981 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: internal_messages +2024-05-09 14:08:55,981 INFO MainThread:4861 [wandb_run.py:_footer_history_summary_info():3936] rendering history +2024-05-09 14:08:55,982 INFO MainThread:4861 [wandb_run.py:_footer_history_summary_info():3968] rendering summary +2024-05-09 14:08:55,983 DEBUG HandlerThread:4861 [handler.py:handle_request():146] handle_request: shutdown +2024-05-09 14:08:55,983 INFO HandlerThread:4861 [handler.py:finish():866] shutting down handler +2024-05-09 14:08:56,980 INFO WriterThread:4861 [datastore.py:close():296] close: runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/run-vqw25nkx.wandb +2024-05-09 14:08:56,980 INFO SenderThread:4861 [sender.py:finish():1546] shutting down sender diff --git a/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug.log b/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..20f16f66df3d908b0be35c3bfcb0701e7bc181c6 --- /dev/null +++ b/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug.log @@ -0,0 +1,27 @@ +2024-05-09 14:01:23,067 INFO MainThread:4587 [wandb_setup.py:_flush():76] Current SDK version is 0.16.6 +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Configure stats pid to 4587 +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Loading settings from /home/users/sschmidg/.config/wandb/settings +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Loading settings from /scratch/groups/willhies/sschmidg/prismatic-vlms/wandb/settings +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Loading settings from environment variables: {} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': 'scripts/pretrain.py', 'program_abspath': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py', 'program': '/scratch/groups/willhies/sschmidg/prismatic-vlms/scripts/pretrain.py'} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Applying login settings: {} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_setup.py:_flush():76] Applying login settings: {'mode': 'offline'} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:_log_setup():521] Logging user logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug.log +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:_log_setup():522] Logging internal logs to runs/med-instruct+testmodel+stage-finetune+x7/wandb/offline-run-20240509_140123-vqw25nkx/logs/debug-internal.log +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():561] calling init triggers +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():568] wandb.init called with sweep_config: {} +config: {'model': {'type': 'one-stage+7b', 'model_id': 'testmodel', 'arch_specifier': 'no-align+gelu-mlp', 'vision_backbone_id': 'clip-vit-b', 'llm_backbone_id': 'llama2-7b-chat', 'image_resize_strategy': 'letterbox', 'llm_max_length': 2048, 'align_epochs': 1, 'align_max_steps': None, 'align_global_batch_size': 256, 'align_per_device_batch_size': 16, 'align_learning_rate': 0.001, 'align_weight_decay': 0.0, 'align_max_grad_norm': 1.0, 'align_lr_scheduler_type': 'linear-warmup+cosine-decay', 'align_warmup_ratio': 0.03, 'align_train_strategy': 'fsdp-shard-grad-op', 'finetune_epochs': 1, 'finetune_max_steps': None, 'finetune_global_batch_size': 128, 'finetune_per_device_batch_size': 16, 'finetune_learning_rate': 2e-05, 'finetune_weight_decay': 0.1, 'finetune_max_grad_norm': 1.0, 'finetune_lr_scheduler_type': 'linear-warmup+cosine-decay', 'finetune_warmup_ratio': 0.03, 'finetune_train_strategy': 'fsdp-full-shard', 'enable_gradient_checkpointing': True, 'enable_mixed_precision_training': True, 'reduce_in_full_precision': False}, 'dataset': {'type': 'med-instruct', 'dataset_id': 'med-instruct', 'align_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'finetune_stage_components': ['download/med-v0.1-instruct/med_v0_1_mix.json', 'download/med-v0.1-instruct'], 'dataset_root_dir': 'data'}, 'stage': 'finetune', 'pretrained_checkpoint': None, 'run_id': 'med-instruct+testmodel+stage-finetune+x7', 'run_root_dir': 'runs', 'seed': 7, 'trackers': ['jsonl', 'wandb'], 'wandb_project': 'onyx-vlms', 'wandb_entity': 'stanford-voltron'} +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():611] starting backend +2024-05-09 14:01:23,068 INFO MainThread:4587 [wandb_init.py:init():615] setting up manager +2024-05-09 14:01:23,073 INFO MainThread:4587 [backend.py:_multiprocessing_setup():105] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2024-05-09 14:01:23,075 INFO MainThread:4587 [wandb_init.py:init():623] backend started and connected +2024-05-09 14:01:23,086 INFO MainThread:4587 [wandb_init.py:init():715] updated telemetry +2024-05-09 14:01:23,094 INFO MainThread:4587 [wandb_init.py:init():748] communicating run to backend with 90.0 second timeout +2024-05-09 14:01:23,099 INFO MainThread:4587 [wandb_init.py:init():799] starting run threads in backend +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_console_start():2335] atexit reg +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_redirect():2190] redirect: wrap_raw +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_redirect():2255] Wrapping output streams. +2024-05-09 14:01:30,270 INFO MainThread:4587 [wandb_run.py:_redirect():2280] Redirects installed. +2024-05-09 14:01:30,271 INFO MainThread:4587 [wandb_init.py:init():842] run started, returning control to user process +2024-05-09 14:08:56,984 WARNING MsgRouterThr:4587 [router.py:message_loop():77] message_loop has been closed diff --git a/wandb/offline-run-20240509_140123-vqw25nkx/run-vqw25nkx.wandb b/wandb/offline-run-20240509_140123-vqw25nkx/run-vqw25nkx.wandb new file mode 100644 index 0000000000000000000000000000000000000000..15b05152a0b8003cfd0f404922ca728881b09a02 Binary files /dev/null and b/wandb/offline-run-20240509_140123-vqw25nkx/run-vqw25nkx.wandb differ