| wandb_project: joy-caption-1 |
| device_batch_size: 2 |
| batch_size: 256 |
| learning_rate: 0.0002 |
| warmup_samples: 18000 |
| max_samples: 600000 |
| save_every: 50000 |
| test_every: 50000 |
| use_amp: true |
| grad_scaler: true |
| lr_scheduler_type: cosine |
| min_lr_ratio: 0.0 |
| allow_tf32: true |
| seed: 69 |
| num_workers: 8 |
| optimizer_type: adamw |
| adam_beta1: 0.9 |
| adam_beta2: 0.999 |
| adam_eps: 1.0e-08 |
| adam_weight_decay: 0.0 |
| clip_grad_norm: 1.0 |
| dataset: fancyfeast/joy-captioning-20240924a |
| clip_model: google/siglip-so400m-patch14-384 |
| text_model: ../lora-train/lora_model_vwbzycxh |
| resume: null |
| gradient_checkpointing: false |
| test_size: 2048 |
| grad_scaler_init: 65536.0 |
| max_caption_length: 257 |
| num_image_tokens: 32 |
| adapter_type: mlp |
| text_model_dtype: bfloat16 |
| pre_test: false |
| train_image_model: true |
| image_model_lr: null |
| train_lora: true |
| lora_r: 64 |
| lora_alpha: 16 |
| lora_dropout: 0.1 |
|
|
