sft_config.yaml

# ─────────────────────────────────────────────────────────────────────────────
#  LUNA 100M — SFT Fine-Tuning Configuration
# ─────────────────────────────────────────────────────────────────────────────

auto_config: true

# ── Paths ─────────────────────────────────────────────────────────────────────
hf_model_repo:   "ASTERIZER/LUNA-100M"         # HF model repo with pretrained latest.pt
hf_dataset_repo: "ASTERIZER/Luna_Dataset"       # HF dataset repo with SFT data
pretrained_ckpt: "Base/out/pretrain/luna_100m/latest.pt"  # local path (auto-downloaded)
train_json:      "Base/Datasets/sft_clean/train.json"
val_json:        "Base/Datasets/sft_clean/val.json"
out_dir:         "Base/out/sft/luna_100m_sft"
tokenizer_dir:   "Base/checkpoints/EleutherAI/pythia-160m"

# ── Model (must match pretrained — do NOT change) ────────────────────────────
model:
  vocab_size: 50304
  seq_len:    1024
  n_layer:    10
  n_embd:     768
  n_head:     12

# ── Training ──────────────────────────────────────────────────────────────────
train:
  epochs:           2
  max_tokens:       0            # 0 = use epochs instead
  lr_warmup_steps:  200
  save_interval:    500
  log_interval:     10
  eval_interval:    500          # run validation + eval prompts every N steps
  max_norm:         1.0

# ── Optimizer (lower LR for fine-tuning) ──────────────────────────────────────
optimizer:
  lr:           1.5e-5
  min_lr:       1.0e-6
  weight_decay: 0.01
  betas:        [0.9, 0.95]
  eps:          1.0e-8

# ── Batch sizing ──────────────────────────────────────────────────────────────
batch:
  global_batch: 64
  micro_batch:  8
  grad_accum:   8

# ── DataLoader ────────────────────────────────────────────────────────────────
dataloader:
  num_workers: 4
  pin_memory:  true

# ── Hardware ──────────────────────────────────────────────────────────────────
hardware:
  precision: "bf16"
  compile:   false

# ── Eval prompts (run after each checkpoint) ──────────────────────────────────
eval_prompts:
  - "Who are you?"
  - "What is your name?"
  - "Who created you?"
  - "Who is Asterizer?"
  - "What is the capital of France?"
  - "Explain photosynthesis in simple terms."
  - "Write a short poem about the moon."
  - "What is 25 times 4?"
  - "Translate 'hello' to Spanish."
  - "Summarize the theory of evolution in two sentences."