| # ───────────────────────────────────────────────────────────────────────────── | |
| # LUNA 100M — Training Configuration | |
| # Single source of truth for all hyperparameters. | |
| # | |
| # auto_config: true → All batch/LR/worker settings are auto-detected from | |
| # available VRAM / RAM / CPU at runtime. Your values below | |
| # are used as FALLBACKS only if detection fails. | |
| # | |
| # auto_config: false → Every value below is used as-is. Nothing is overridden. | |
| # Use this when you've already benchmarked and want | |
| # repeatable, fixed runs. | |
| # ───────────────────────────────────────────────────────────────────────────── | |
| auto_config: true # ← flip to false to lock everything below | |
| # ── Data ────────────────────────────────────────────────────────────────────── | |
| data_path: "Base/data/litdata_pretrain_final" # local default; overridden by --data_path | |
| out_dir: "out/pretrain/luna-100m" | |
| tokenizer_dir: "Base/checkpoints/EleutherAI/pythia-160m" | |
| # ── Model (fixed for LUNA-100M — do not change) ─────────────────────────────── | |
| model: | |
| vocab_size: 50304 # ceil(50277/128)*128 — pythia tokenizer with EOS padding | |
| seq_len: 1024 | |
| n_layer: 10 | |
| n_embd: 768 | |
| n_head: 12 | |
| # ── Training budget ─────────────────────────────────────────────────────────── | |
| train: | |
| max_tokens: 4515286950 # full dataset (verified from index.json, 270 chunks) | |
| lr_warmup_steps: 500 # [AUTO] scaled to 5% of total_steps if auto_config | |
| save_interval: 1000 # save checkpoint every N optimizer steps | |
| log_interval: 10 # print log every N steps | |
| max_norm: 1.0 # gradient clip norm | |
| # ── Optimiser ───────────────────────────────────────────────────────────────── | |
| optimizer: | |
| lr: 0.0006 # 6e-4 [AUTO] scaled by sqrt(global_batch/120) if auto_config | |
| min_lr: 0.00006 # 6e-5 | |
| weight_decay: 0.1 | |
| betas: [0.9, 0.95] | |
| eps: 1.0e-8 | |
| # ── Batch sizing ────────────────────────────────────────────────────────────── | |
| # When auto_config: true → micro_batch and grad_accum are ignored; the script | |
| # probes VRAM and fills it to ~82% saturation, then | |
| # computes grad_accum to hit global_batch. | |
| # When auto_config: false → micro_batch × grad_accum must equal global_batch. | |
| batch: | |
| global_batch: 120 # target total samples per optimizer step | |
| micro_batch: 12 # [MANUAL] samples per GPU forward pass (ignored when auto) | |
| grad_accum: 10 # [MANUAL] accumulation steps (ignored when auto) | |
| # ── DataLoader ──────────────────────────────────────────────────────────────── | |
| # When auto_config: true → num_workers auto = cpu_cores // 2, capped by RAM | |
| # When auto_config: false → num_workers used as-is | |
| dataloader: | |
| num_workers: -1 # -1 = auto; set to 0 to disable multiprocessing | |
| pin_memory: true # [AUTO] disabled if RAM < 16GB | |
| # ── Hardware / precision ────────────────────────────────────────────────────── | |
| # When auto_config: true → precision detected from GPU compute capability | |
| # When auto_config: false → use the value below | |
| hardware: | |
| precision: "bf16" # bf16 | fp16 | fp32 | |
| compile: true # torch.compile (requires Triton — Linux/cloud only) | |