feat: train_hyper_loop with progressive looping, evolution loss feedback, no progressive_unfreeze default\n\nActivates dormant ch1mera paradigms:\n1. Progressive looping: 1→2→3 Parcae loops during training\n2. Evolution receives prev_loss for surprise-based memory writes\n3. progressive_unfreeze disabled by default (all layers train from start)\n4. Logs loop count and NaN-safe averaging"

chimera/training/loops.py

@@ -10,6 +10,7 @@ import torch
 import chimera_turbo
 
 from .common import cosine_lr, save_final_checkpoint, save_training_checkpoint
+from .hyper import ProgressiveLoopScheduler
 
 
 def train_fast_loop(args, model, config, loader, compute_loss) -> str:
@@ -149,18 +150,20 @@ def train_standard_loop(args, model, config, loader, compute_loss, optimizer, us
 def train_hyper_loop(args, model, config, dataset, initial_seq, grow, unfreezer):
     use_compile = getattr(args, "compile", False)
 
-    # Let chimera_turbo.apply() use its v8 BitNet-aligned defaults:
-    # lr=1.5e-3, wd=0.01, β=(0.9,0.98), warmup=750
-    # Only override lr if the user explicitly set it via CLI
     model, optimizer, scheduler = chimera_turbo.apply(
         model,
         max_steps=args.max_steps,
-        lr=args.lr,               # CLI default is 1e-3; chimera_turbo default is 1.5e-3
-        # weight_decay and warmup_steps use chimera_turbo defaults (0.01, 750)
+        lr=args.lr,
         use_compile=use_compile,
         use_ipex=True,
     )
     model.train()
+
+    # ── Progressive looping: 1→2→3 Parcae loops ──
+    loop_sched = ProgressiveLoopScheduler(args.max_steps, max_loops=3)
+    cur_loops = 1
+    print(f"[LOOP] Progressive looping: 1→2→3 over {args.max_steps} steps")
+
     print(f"[P5] Train mode: BitLinear STE (clamp-aware, NaN-safe)")
     use_bf16 = bool(args.bf16)
 
@@ -168,6 +171,7 @@ def train_hyper_loop(args, model, config, dataset, initial_seq, grow, unfreezer)
     log_f = open(os.path.join(args.output_dir, "log_hyper.jsonl"), "w")
     step = 0
     total_loss = 0.0
+    valid_loss_count = 0
     best_loss = float("inf")
     toks = 0
     t0 = time.time()
@@ -179,10 +183,11 @@ def train_hyper_loop(args, model, config, dataset, initial_seq, grow, unfreezer)
     data_iter = iter(loader)
 
     print(f"\n{'=' * 65}")
-    print(f"Training  eff_batch={eff_batch}  seq={cur_seq}")
+    print(f"Training  eff_batch={eff_batch}  seq={cur_seq}  loops={cur_loops}")
     print(f"{'=' * 65}\n")
 
     while step < args.max_steps:
+        # ── GrowLength seq scheduling ──
         if grow:
             ns = grow.get_seq_len(step)
             if ns != cur_seq:
@@ -194,8 +199,21 @@ def train_hyper_loop(args, model, config, dataset, initial_seq, grow, unfreezer)
                 )
                 data_iter = iter(loader)
                 print(f"  [P1] seq → {cur_seq}  batch → {eff_batch}")
+
+        # ── Progressive loop scheduling ──
+        new_loops = loop_sched.get_loops(step)
+        if new_loops != cur_loops:
+            cur_loops = new_loops
+            if hasattr(model, "loop_controller"):
+                model.loop_controller.loop_default = cur_loops
+            elif hasattr(model, "_orig_mod") and hasattr(model._orig_mod, "loop_controller"):
+                model._orig_mod.loop_controller.loop_default = cur_loops
+            print(f"  [LOOP] loops → {cur_loops}")
+
+        # ── Progressive unfreeze (if enabled) ──
         if unfreezer:
             unfreezer.update(step)
+
         try:
             batch = next(data_iter)
         except StopIteration:
@@ -209,31 +227,36 @@ def train_hyper_loop(args, model, config, dataset, initial_seq, grow, unfreezer)
         )
 
         cur_lr = optimizer.param_groups[0]["lr"]
-        total_loss += loss_val if math.isfinite(loss_val) else 0.0
+        if math.isfinite(loss_val):
+            total_loss += loss_val
+            valid_loss_count += 1
         toks += batch["input_ids"].numel()
         step += 1
 
         if step % args.log_every == 0:
             dt = time.time() - t0
-            avg = total_loss / args.log_every
+            avg = total_loss / max(1, valid_loss_count)
             ppl = math.exp(min(avg, 20)) if math.isfinite(avg) else float("nan")
             tps = toks / dt if dt > 0 else 0
             eta = (args.max_steps - step) / (step / dt) / 3600 if dt > 0 else 0
             log_f.write(
                 json.dumps({
-                    "step": step, "loss": round(avg, 4) if math.isfinite(avg) else None,
+                    "step": step,
+                    "loss": round(avg, 4) if math.isfinite(avg) else None,
                     "ppl": round(ppl, 2) if math.isfinite(ppl) else None,
                     "lr": round(cur_lr, 6), "tok/s": round(tps),
                     "seq_len": cur_seq, "eff_batch": eff_batch,
+                    "loops": cur_loops,
                 }) + "\n"
             )
             log_f.flush()
             print(
                 f"  step {step:>6}/{args.max_steps} | loss {avg:.4f} | ppl {ppl:>8.2f} "
-                f"| lr {cur_lr:.2e} | {tps:,.0f} tok/s | seq {cur_seq} | ETA {eta:.1f}h"
+                f"| lr {cur_lr:.2e} | {tps:,.0f} tok/s | seq {cur_seq} | L{cur_loops} | ETA {eta:.1f}h"
             )
             best_loss = min(best_loss, avg) if math.isfinite(avg) else best_loss
             total_loss = 0.0
+            valid_loss_count = 0
             toks = 0
             t0 = time.time()