Upload folder using huggingface_hub

train/curate_pivot_set.py

@@ -63,12 +63,19 @@ def cmd_baseline(args: argparse.Namespace) -> None:
     )
     model.eval()
 
+    # Neutral prompt: avoid the word "sarcasm" before the model has answered,
+    # list "sincere" before "sarcastic" to fight the prefix bias of Qwen2.5-3B.
     PROMPT = (
-        "You are an expert at detecting sarcasm in dialogue.\n"
-        "Read the conversational context and the target line.\n"
-        "Answer with EXACTLY one of: sarcastic | sincere\n"
-        "Then on the next line, your confidence as a float 0..1.\n\n"
-        "{ctx}\n\nTarget:\n[{spk}] {utt}\n\nAnswer:"
+        "You will read a line of TV dialogue with its conversational context.\n"
+        "Decide whether the speaker is being sincere (means what they say) "
+        "or sarcastic (means the opposite of what they say).\n\n"
+        "{ctx}\n\n"
+        "Target line:\n[{spk}] {utt}\n\n"
+        "Output exactly two lines, in this format:\n"
+        "Label: sincere\n"
+        "Confidence: 0.7\n\n"
+        "Now classify the target line above.\n"
+        "Output:\n"
     )
 
     out: Dict[str, Dict] = {}
@@ -95,19 +102,25 @@ def cmd_baseline(args: argparse.Namespace) -> None:
             )
         text = tok.decode(o[0][ids.input_ids.shape[1]:], skip_special_tokens=True).strip()
 
-        first_word = text.lower().split()[0] if text.strip() else ""
-        pred_label = "sarcastic" if "sarc" in first_word else "sincere"
-        # Try to extract confidence
+        # Parse "Label: X\nConfidence: Y" format
+        pred_label = "sincere"   # default to sincere if parsing fails (less biased)
         conf = 0.5
         for line in text.splitlines():
-            line = line.strip()
-            try:
-                v = float(line)
-                if 0.0 <= v <= 1.0:
-                    conf = v
-                    break
-            except ValueError:
-                continue
+            stripped = line.strip().lower()
+            if stripped.startswith("label:"):
+                value = stripped[len("label:"):].strip()
+                if "sarc" in value:
+                    pred_label = "sarcastic"
+                elif "sinc" in value:
+                    pred_label = "sincere"
+            elif stripped.startswith("confidence:"):
+                value = stripped[len("confidence:"):].strip()
+                try:
+                    v = float(value)
+                    if 0.0 <= v <= 1.0:
+                        conf = v
+                except ValueError:
+                    pass
         out[clip_id] = {
             "predicted": pred_label,
             "confidence": conf,

train/sft_warmup.py

@@ -0,0 +1,329 @@
+"""SFT warmup: bootstrap the format + reasoning skeleton before GRPO.
+
+Why: GRPO from a base model spends ~50-100 steps just learning to emit
+`<think>...</think><final>{...}</final>`. SFT on ~100 ideal completions
+gets the format perfect upfront so all GRPO steps focus on improving
+correctness.
+
+How: we synthesize "ideal" completions deterministically from the gold label
++ actual prosody features. No API call. The reasoning text references the
+real prosody numbers. The final tag uses the gold label.
+
+This is a min-cost run that:
+  - Generates 100 (prompt, ideal) pairs locally
+  - SFTs Qwen2.5-3B + LoRA for 1 epoch (~5-10 min on L4)
+  - Saves the LoRA checkpoint to HF Hub
+  - Prints 3 before/after sample completions for visual inspection
+
+Output: LoRA adapter pushed to HF Hub at aamrinder/subtext-arena-sft
+        (then GRPO Run #1 starts FROM this adapter, not from vanilla Qwen)
+"""
+from __future__ import annotations
+
+import json
+import os
+import random
+import sys
+from pathlib import Path
+from typing import Dict, List
+
+ROOT = Path(__file__).resolve().parent.parent
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+try:
+    from subtext_arena.server.scenarios import load_scenarios
+    from subtext_arena.train.train_grpo import (
+        SYSTEM_PROMPT, build_full_observation,
+    )
+except ImportError:
+    from server.scenarios import load_scenarios
+    from train.train_grpo import SYSTEM_PROMPT, build_full_observation
+
+
+# ---------------------------------------------------------------------------
+# Generate ideal completions deterministically from gold + prosody
+# ---------------------------------------------------------------------------
+
+POSITIVE_WORDS = {
+    "great", "wonderful", "amazing", "perfect", "love", "brilliant",
+    "privilege", "lovely", "fantastic", "excellent", "fabulous", "delighted",
+    "thrilled", "grateful", "honored", "blessed",
+}
+
+
+def synth_ideal_completion(scenario: Dict) -> str:
+    """Build a high-quality (think + final) target completion for SFT.
+
+    Reasoning is grounded in the actual prosody numbers and gold label.
+    """
+    gold = "sarcastic" if scenario["sarcasm"] else "sincere"
+    prosody = scenario.get("prosody") or {}
+    utt = scenario.get("utterance", "").lower()
+
+    pitch_var = float(prosody.get("pitch_var_hz", 0.0))
+    pitch_mean = float(prosody.get("pitch_mean_hz", 0.0))
+    pre_pause = int(prosody.get("pre_pause_ms", 0))
+    pauses = prosody.get("pauses", []) or []
+    voiced_ratio = float(prosody.get("voiced_ratio", 0.0))
+    n_internal = len(pauses)
+
+    parts = []
+
+    # 1) Lexical observation
+    pos_count = sum(1 for w in utt.split() if w.strip(".,!?'\"") in POSITIVE_WORDS)
+    if pos_count > 0:
+        parts.append(
+            f"The literal lexical content is positive ({pos_count} positive word"
+            f"{'s' if pos_count > 1 else ''}: {', '.join(w for w in utt.split() if w.strip(chr(46)+chr(44)+chr(33)+chr(63)+chr(39)+chr(34)) in POSITIVE_WORDS)[:120]})."
+        )
+    elif "?" in utt:
+        parts.append("The line is phrased as a question.")
+    else:
+        parts.append("The lexical content is neutral or descriptive.")
+
+    # 2) Prosody observation (only if features are reliable)
+    if voiced_ratio < 0.1:
+        parts.append(
+            "Prosody is unreliable for this clip (low voiced-frame ratio). "
+            "Lexical and contextual cues should dominate."
+        )
+        prosody_evidence = "weak"
+    else:
+        if pitch_var > 45:
+            parts.append(
+                f"Pitch variability is HIGH ({pitch_var:.0f} Hz over voiced frames), "
+                "suggesting exaggerated melodic delivery."
+            )
+            prosody_evidence = "exaggerated"
+        elif pitch_var < 25:
+            parts.append(
+                f"Pitch variability is LOW ({pitch_var:.0f} Hz), suggesting "
+                "flat or minimally inflected delivery."
+            )
+            prosody_evidence = "flat"
+        else:
+            parts.append(
+                f"Pitch variability is moderate ({pitch_var:.0f} Hz), neither "
+                "flat nor exaggerated."
+            )
+            prosody_evidence = "moderate"
+
+        if pre_pause >= 250:
+            parts.append(
+                f"There is a {pre_pause}ms pre-utterance pause — speakers often "
+                "use such setup pauses for ironic or emphatic delivery."
+            )
+        if n_internal >= 1:
+            parts.append(
+                f"There {'is' if n_internal == 1 else 'are'} {n_internal} internal pause"
+                f"{'' if n_internal == 1 else 's'} >150ms within the utterance."
+            )
+
+    # 3) Conclusion grounded in the evidence
+    if gold == "sarcastic":
+        if pos_count > 0 and prosody_evidence == "exaggerated":
+            parts.append(
+                "Positive lexical content combined with exaggerated melodic "
+                "delivery is the signature pattern of sarcastic delivery — "
+                "the words say one thing, the prosody says the opposite."
+            )
+        elif prosody_evidence == "exaggerated":
+            parts.append(
+                "Exaggerated prosodic shape on otherwise non-emphatic content "
+                "is consistent with mock or ironic delivery."
+            )
+        else:
+            parts.append(
+                "Subtle cues taken together (delivery, emphasis pause, "
+                "context) suggest the speaker is being ironic rather than literal."
+            )
+    else:
+        if prosody_evidence == "flat":
+            parts.append(
+                "Flat prosodic delivery on neutral or genuine content "
+                "indicates the speaker means what they say."
+            )
+        elif pos_count > 0 and prosody_evidence != "exaggerated":
+            parts.append(
+                "Positive lexical content paired with non-exaggerated delivery "
+                "indicates sincere expression."
+            )
+        else:
+            parts.append(
+                "Lacking strong markers of irony, the speaker appears to be "
+                "expressing genuine intent."
+            )
+
+    think_text = " ".join(parts)
+    final_json = json.dumps({"label": gold, "confidence": 0.85}, separators=(",", ":"))
+    return f"<think>\n{think_text}\n</think>\n<final>{final_json}</final>"
+
+
+def build_sft_dataset(scenarios, n_rows: int = 100, seed: int = 0):
+    """Pick n_rows clips, build (prompt, ideal_completion) pairs."""
+    from datasets import Dataset
+
+    rng = random.Random(seed)
+    # Balance classes in the SFT set
+    sarc_ids = [k for k, v in scenarios.items() if v["sarcasm"]]
+    sinc_ids = [k for k, v in scenarios.items() if not v["sarcasm"]]
+    rng.shuffle(sarc_ids); rng.shuffle(sinc_ids)
+    chosen = sarc_ids[: n_rows // 2] + sinc_ids[: n_rows - n_rows // 2]
+    rng.shuffle(chosen)
+
+    rows = []
+    for cid in chosen:
+        sc = scenarios[cid]
+        user_text = build_full_observation(cid, scenarios)
+        ideal = synth_ideal_completion(sc)
+        # Use the chat-completion format — Qwen2.5-Instruct expects this
+        rows.append({
+            "messages": [
+                {"role": "system", "content": SYSTEM_PROMPT},
+                {"role": "user", "content": user_text},
+                {"role": "assistant", "content": ideal},
+            ],
+            "clip_id": cid,
+            "gold": "sarcastic" if sc["sarcasm"] else "sincere",
+        })
+    return Dataset.from_list(rows)
+
+
+def sample_before_after(model, tokenizer, scenarios, sample_clip_ids, label_for_log: str):
+    """Generate completions on a few held-out clips for visual inspection."""
+    print(f"\n----- Sample completions ({label_for_log}) -----")
+    for cid in sample_clip_ids:
+        sc = scenarios[cid]
+        gold = "sarcastic" if sc["sarcasm"] else "sincere"
+        messages = [
+            {"role": "system", "content": SYSTEM_PROMPT},
+            {"role": "user", "content": build_full_observation(cid, scenarios)},
+        ]
+        inputs = tokenizer.apply_chat_template(
+            messages, return_tensors="pt", add_generation_prompt=True
+        ).to(model.device)
+        out = model.generate(
+            inputs, max_new_tokens=350, do_sample=True, temperature=0.7,
+            pad_token_id=tokenizer.eos_token_id,
+        )
+        text = tokenizer.decode(out[0][inputs.shape[1]:], skip_special_tokens=True)
+        print(f"\nClip {cid} (gold={gold}, speaker={sc.get('speaker')}):")
+        print(text[:1000])
+        print("---")
+
+
+# ---------------------------------------------------------------------------
+# Main
+# ---------------------------------------------------------------------------
+
+def main():
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--model", default="Qwen/Qwen2.5-3B-Instruct")
+    parser.add_argument("--n-rows", type=int, default=100)
+    parser.add_argument("--epochs", type=int, default=1)
+    parser.add_argument("--lora-r", type=int, default=8)
+    parser.add_argument("--learning-rate", type=float, default=2e-4)
+    parser.add_argument("--output-dir", default="/tmp/sft_warmup")
+    parser.add_argument("--push-to-hub", default=None,
+                        help="If set, e.g. 'aamrinder/subtext-arena-sft', push the LoRA there")
+    parser.add_argument("--n-sample-clips", type=int, default=3,
+                        help="How many clips to generate before/after samples on")
+    args = parser.parse_args()
+
+    print(f"[load-scenarios]")
+    scenarios = load_scenarios()
+    print(f"   {len(scenarios)} clips")
+
+    print(f"[build-sft-dataset] n_rows={args.n_rows}")
+    ds = build_sft_dataset(scenarios, n_rows=args.n_rows)
+    print(f"   {len(ds)} (prompt, ideal-completion) pairs")
+    print(f"   first ideal completion preview:")
+    first_msgs = ds[0]["messages"]
+    print("     " + first_msgs[-1]["content"].replace("\n", "\n     ")[:400])
+
+    # Pick held-out clips for before/after sampling
+    sample_ids = [k for k in scenarios.keys() if k not in {r["clip_id"] for r in ds}][: args.n_sample_clips]
+
+    # Load model
+    print(f"\n[load-model] {args.model}, 4-bit + LoRA")
+    import torch as _t
+    from transformers import (
+        AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig,
+    )
+    from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+    from trl import SFTTrainer, SFTConfig
+
+    bnb = BitsAndBytesConfig(
+        load_in_4bit=True,
+        bnb_4bit_compute_dtype=_t.bfloat16,
+        bnb_4bit_quant_type="nf4",
+        bnb_4bit_use_double_quant=True,
+    )
+    tokenizer = AutoTokenizer.from_pretrained(args.model)
+    if tokenizer.pad_token is None:
+        tokenizer.pad_token = tokenizer.eos_token
+    base = AutoModelForCausalLM.from_pretrained(
+        args.model, quantization_config=bnb, dtype=_t.bfloat16, device_map="auto",
+    )
+    base = prepare_model_for_kbit_training(base, use_gradient_checkpointing=True)
+    peft_config = LoraConfig(
+        r=args.lora_r, lora_alpha=args.lora_r, lora_dropout=0.0, bias="none",
+        target_modules=["q_proj", "k_proj", "v_proj", "o_proj"],
+        task_type="CAUSAL_LM",
+    )
+    model = get_peft_model(base, peft_config)
+
+    # Sample BEFORE training
+    sample_before_after(model, tokenizer, scenarios, sample_ids, "BEFORE SFT")
+
+    # SFT training
+    print(f"\n[sft-train] {args.epochs} epoch(s), lr={args.learning_rate}")
+    config = SFTConfig(
+        output_dir=args.output_dir,
+        num_train_epochs=args.epochs,
+        per_device_train_batch_size=2,
+        gradient_accumulation_steps=4,
+        learning_rate=args.learning_rate,
+        bf16=True,
+        gradient_checkpointing=True,
+        logging_steps=2,
+        save_strategy="no",
+        report_to="none",
+        max_length=4096,
+    )
+    trainer = SFTTrainer(
+        model=model,
+        args=config,
+        train_dataset=ds,
+        processing_class=tokenizer,
+    )
+    trainer.train()
+    trainer.save_model(args.output_dir)
+    print(f"\n[done] LoRA adapter saved to {args.output_dir}")
+
+    # Sample AFTER training
+    sample_before_after(model, tokenizer, scenarios, sample_ids, "AFTER SFT")
+
+    # Optional: push to HF Hub
+    if args.push_to_hub:
+        from huggingface_hub import HfApi
+        api = HfApi()
+        # Create the repo first (idempotent)
+        try:
+            api.create_repo(repo_id=args.push_to_hub, repo_type="model", exist_ok=True)
+        except Exception as e:
+            print(f"[warn] create_repo: {e}")
+        # Upload the LoRA adapter directory
+        api.upload_folder(
+            folder_path=args.output_dir,
+            repo_id=args.push_to_hub,
+            repo_type="model",
+            commit_message=f"SFT warmup checkpoint ({args.n_rows} examples, {args.epochs} epoch)",
+        )
+        print(f"[done] LoRA adapter pushed to https://huggingface.co/{args.push_to_hub}")
+
+
+if __name__ == "__main__":
+    main()