aethera-gp
/

kotodama-108m-instruct

+model_config_path: /home/athuser/luxi-files/kotodama/configs/model.yaml
+model_config_section: proxy
+attn_res: true
+attn_res_boundaries:
+- 0
+- 3
+- 7
+- 12
+- 21
+- 25
+dataset: pipeline/formatted/tokenized/train
+eval_dataset: pipeline/formatted/tokenized/eval
+packed: true
+max_seq_len: 4096
+batch_size: 4
+gradient_accumulation: 1
+max_steps: -1
+bf16: true
+max_grad_norm: 1.0
+muon_momentum: 0.95
+muon_weight_decay: 0.01
+muon_ns_iterations: 5
+muon_ns_coefficients: gram_ns
+adamw_betas:
+- 0.9
+- 0.95
+adamw_weight_decay: 0.1
+warmup_ratio: 0.05
+wsd_decay_start: 1.0
+wsd_decay_type: sqrt
+logging_steps: 10
+grad_analysis_every: 10
+weight_drift_every: 10
+geo_tier1_every: 10
+sample_every: 50
+save_every: 25
+eval_steps: 25
+checkpoint_keep: 3
+async_save: true
+checkpoint_compress: true
+checkpoint_shm_dir: /dev/shm/luxia-sft-ckpts
+wandb_project: kotodama-sft-sweep
+wandb_entity: aethera
+num_workers: 4
+pretrained_checkpoint: /home/athuser/luxi-files/kotodama/checkpoints/books-cpt/step_00017336.pt.zst
+muon_lr: 0.003
+adamw_lr: 0.00030000000000000003
+num_epochs: 2
+wandb_run_name: sweep-books-cpt-lr3e-03-ep2
+output_dir: outputs/sweep/sweep-books-cpt-lr3e-03-ep2
+checkpoint_dir: outputs/sweep/sweep-books-cpt-lr3e-03-ep2/checkpoints