Upload sft_config.yaml with huggingface_hub

sft_config.yaml

@@ -0,0 +1,69 @@
+# ─────────────────────────────────────────────────────────────────────────────
+#  LUNA 100M — SFT Fine-Tuning Configuration
+# ─────────────────────────────────────────────────────────────────────────────
+
+auto_config: true
+
+# ── Paths ─────────────────────────────────────────────────────────────────────
+hf_model_repo:   "ASTERIZER/LUNA-100M"         # HF model repo with pretrained latest.pt
+hf_dataset_repo: "ASTERIZER/Luna_Dataset"       # HF dataset repo with SFT data
+pretrained_ckpt: "Base/out/pretrain/luna_100m/latest.pt"  # local path (auto-downloaded)
+train_json:      "Base/Datasets/sft_clean/train.json"
+val_json:        "Base/Datasets/sft_clean/val.json"
+out_dir:         "Base/out/sft/luna_100m_sft"
+tokenizer_dir:   "Base/checkpoints/EleutherAI/pythia-160m"
+
+# ── Model (must match pretrained — do NOT change) ────────────────────────────
+model:
+  vocab_size: 50304
+  seq_len:    1024
+  n_layer:    10
+  n_embd:     768
+  n_head:     12
+
+# ── Training ──────────────────────────────────────────────────────────────────
+train:
+  epochs:           2
+  max_tokens:       0            # 0 = use epochs instead
+  lr_warmup_steps:  200
+  save_interval:    500
+  log_interval:     10
+  eval_interval:    500          # run validation + eval prompts every N steps
+  max_norm:         1.0
+
+# ── Optimizer (lower LR for fine-tuning) ──────────────────────────────────────
+optimizer:
+  lr:           1.5e-5
+  min_lr:       1.0e-6
+  weight_decay: 0.01
+  betas:        [0.9, 0.95]
+  eps:          1.0e-8
+
+# ── Batch sizing ──────────────────────────────────────────────────────────────
+batch:
+  global_batch: 64
+  micro_batch:  8
+  grad_accum:   8
+
+# ── DataLoader ────────────────────────────────────────────────────────────────
+dataloader:
+  num_workers: 4
+  pin_memory:  true
+
+# ── Hardware ──────────────────────────────────────────────────────────────────
+hardware:
+  precision: "bf16"
+  compile:   false
+
+# ── Eval prompts (run after each checkpoint) ──────────────────────────────────
+eval_prompts:
+  - "Who are you?"
+  - "What is your name?"
+  - "Who created you?"
+  - "Who is Asterizer?"
+  - "What is the capital of France?"
+  - "Explain photosynthesis in simple terms."
+  - "Write a short poem about the moon."
+  - "What is 25 times 4?"
+  - "Translate 'hello' to Spanish."
+  - "Summarize the theory of evolution in two sentences."