fix: --all no longer enables progressive_unfreeze (counterproductive with backprop)"

train_hyper.py

@@ -1,27 +1,19 @@
 #!/usr/bin/env python3
 """
-Chimera 5.3 — HYPER CPU Training v3 (10,000+ tok/s target)
-============================================================
+Chimera 5.3 — HYPER CPU Training v4
+=====================================
 
-ALL features preserved: 28 layers, MoE, Parcae looping, SelfEvolution,
-SpanInference, Grammar, EntropyValve, DebtLedger — nothing disabled.
+All ch1mera features active: 28 layers (GD/XM/TM/SK pattern), MoE,
+Parcae looping (progressive 1→2→3), SelfEvolution, SpanInference,
+Grammar, EntropyValve, DebtLedger.
 
-Speed comes from optimizing HOW the forward+MeZO runs, not WHAT it runs:
-
- P1  GrowLength Curriculum     — seq 8→target, huge batch at short lengths
- P2  Reservoir Freezing        — freeze recurrent gates (fewer params to perturb)
- P3  In-Place Seed MeZO       — no randn allocation, seed-replay perturbation
- P4  torch.compile             — fuse ops, eliminate Python overhead
- P5  Train-Mode STE Path      — BitLinear uses STE (no invalidate_packed)
+Training paradigms:
+ P1  GrowLength Curriculum     — seq 8→target, huge batch at short seq
+ P2  Reservoir Freezing        — freeze recurrent gates as random ternary
+ P5  STE + AdamW               — BitNet-paper training (replaces MeZO)
  P6  Aggressive Token Packing  — zero padding waste
- P7  Progressive Unfreeze      — fewer params early = faster perturbation
- P8  Vocab Projection Cache    — cache lm_head weight for 200K vocab
- P9  Loop-1 Training           — force num_loops=1 during training (full arch)
-
-Key insight: MeZO's bottleneck is not the forward pass — it's
-generating+applying random perturbations to 227M params 3× per step.
-Seed-replay MeZO eliminates this entirely: perturb in-place using a
-single seed, replay the same seed to restore/update.
+ P10 Progressive Looping       — Parcae loops 1→2→3 during training
+ P11 NaN-safe training         — skip + recover on gradient explosion
 """
 
 from __future__ import annotations
@@ -72,16 +64,12 @@ def build_model(args):
     return build_model_from_args(args)
 
 
-# ═══════════════════════════════════════════════════════════════════════════
-# MAIN HYPER TRAIN
-# ═══════════════════════════════════════════════════════════════════════════
-
 def train_hyper(args):
     model, config = build_model(args)
     counts = model.count_parameters()
 
     print("=" * 65)
-    print(f"CHIMERA 5.3 HYPER v3 — scale={args.scale}  bf16={args.bf16}")
+    print(f"CHIMERA 5.3 HYPER v4 — scale={args.scale}  bf16={args.bf16}")
     print(f"Layers={config['num_hidden_layers']}  hidden={config['hidden_size']}  "
           f"vocab={config['vocab_size']}  target_seq={args.seq_len}")
     print(f"Threads: {torch.get_num_threads()}  IPEX={_HAS_IPEX}")
@@ -91,12 +79,11 @@ def train_hyper(args):
           f"span={model.span_engine is not None}")
     print("=" * 65)
 
-    # ── P9: Force loop=1 during training ─────────────────────────────
-    # Architecture intact, but save 1 full pass through layers 4-23
+    # ── Parcae: start at 1 loop, progressive scheduler will increase to 2→3
     patch_training_loops(model, num_loops=1)
-    print(f"[P9] Training loops=1 (arch intact, Parcae wired)")
+    print(f"[P10] Progressive looping enabled (1→2→3)")
 
-    # ── P2: Reservoir Freezing ───────────────────────────────────────
+    # ── P2: Reservoir Freezing
     if args.reservoir:
         frozen = apply_reservoir_freezing(model)
         print(f"[P2] Reservoir: froze {frozen:,} gate params")
@@ -104,14 +91,16 @@ def train_hyper(args):
     trainable = sum(p.numel() for p in model.parameters() if p.requires_grad)
     print(f"[INFO] Trainable: {trainable:,} / {counts['total']:,}")
 
-    # ── P7: Progressive Unfreezing ───────────────────────────────────
+    # ── P7: Progressive Unfreezing (OFF by default — counterproductive with backprop)
     unfreezer = None
     if args.progressive_unfreeze:
         unfreezer = ProgressiveUnfreezer(model, args.max_steps, args.unfreeze_stages)
         active = sum(p.numel() for p in model.parameters() if p.requires_grad)
         print(f"[P7] Progressive unfreeze: {active:,} initially trainable")
+    else:
+        print(f"[P7] Progressive unfreeze: OFF (all layers train from start)")
 
-    # ── P1: GrowLength ───────────────────────────────────────────────
+    # ── P1: GrowLength
     if args.growlength:
         stages = [
             (max(8, args.seq_len // 4), 0.30),
@@ -125,7 +114,7 @@ def train_hyper(args):
         grow = None
         initial_seq = args.seq_len
 
-    # ── Data ─────────────────────────────────────────────────────────
+    # ── Data
     tok_budget = args.max_tokens or max(500_000,
         args.max_steps * args.batch_size * (args.seq_len + 1) * 4)
     token_buf = build_token_buffer(
@@ -137,18 +126,14 @@ def train_hyper(args):
     train_hyper_loop(args, model, config, dataset, initial_seq, grow, unfreezer)
 
 
-# ═══════════════════════════════════════════════════════════════════════════
-# CLI
-# ═══════════════════════════════════════════════════════════════════════════
-
 def cli():
-    p = argparse.ArgumentParser(description="Chimera 5.3 HYPER v3")
+    p = argparse.ArgumentParser(description="Chimera 5.3 HYPER v4")
     p.add_argument("--config", default=str(DEFAULT_CONFIG_PATH))
     p.add_argument("--scale", default="tiny", choices=["tiny", "small", "medium", "full"])
     p.add_argument("--seq_len", type=int, default=64)
     p.add_argument("--batch_size", type=int, default=8)
-    p.add_argument("--lr", type=float, default=1e-3)
-    p.add_argument("--warmup", type=int, default=100)
+    p.add_argument("--lr", type=float, default=1.5e-3)  # ← BitNet-interpolated default
+    p.add_argument("--warmup", type=int, default=750)    # ← BitNet paper-exact
     p.add_argument("--max_steps", type=int, default=5000)
     p.add_argument("--max_tokens", type=int, default=None)
     p.add_argument("--max_samples", type=int, default=None)
@@ -168,6 +153,7 @@ def cli():
     g.add_argument("--growlength", action="store_true", default=False)
     g.add_argument("--reservoir", action="store_true", default=False)
     g.add_argument("--mezo-eps", type=float, default=1e-3, dest="mezo_eps")
+    # Progressive unfreeze: OFF by default (counterproductive with backprop)
     g.add_argument("--progressive-unfreeze", action="store_true", default=False,
                    dest="progressive_unfreeze")
     g.add_argument("--unfreeze-stages", type=int, default=4, dest="unfreeze_stages")
@@ -182,11 +168,11 @@ if __name__ == "__main__":
     if args.all:
         args.growlength = True
         args.reservoir = True
-        args.progressive_unfreeze = True
+        # NOTE: progressive_unfreeze deliberately NOT set by --all
+        # It was designed for MeZO and is counterproductive with STE+AdamW
     if args.benchmark:
         args.growlength = True
         args.reservoir = True
-        args.progressive_unfreeze = True
         benchmark_hyper(args)
     else:
         train_hyper(args)