add train_grpo_smoke notebook; quote pip versions in train_grpo

- Smoke notebook: repo setup, imports, TASK_HORIZON=30, one episode, optional ML imports
- Fix zsh redirect bug from unquoted transformers>= in pip cell

Made-with: Cursor

training/train_grpo.ipynb

@@ -25,9 +25,9 @@
       "cell_type": "code",
       "metadata": {},
       "source": [
-        "# Cell 1: Install dependencies\n",
+        "# Cell 1: Install dependencies (quote versions — zsh treats `>` as redirect otherwise)\n",
         "!pip install -q torch torchvision torchaudio\n",
-        "!pip install -q transformers>=4.45.0 accelerate peft>=0.10.0 trl>=0.20.0 datasets bitsandbytes\n",
+        "!pip install -q \"transformers>=4.45.0\" \"accelerate\" \"peft>=0.10.0\" \"trl>=0.20.0\" \"datasets\" \"bitsandbytes\"\n",
         "!pip install -q matplotlib pandas\n",
         "!pip install -q pydantic httpx\n",
         "!pip install -q \"openenv-core[core]>=0.2.2\""
@@ -142,7 +142,7 @@
             "Repo root: /Users/anurag.c/viral-posts-env\n",
             "Working dir: /Users/anurag.c/viral-posts-env\n",
             "Branch: hack1\n",
-            "Commit: b5ad200\n",
+            "Commit: b2fc6b6\n",
             "Plots dir: /Users/anurag.c/viral-posts-env/plots\n"
           ]
         }
@@ -506,27 +506,14 @@
         "if torch.cuda.is_available():\n",
         "    print(f\"CUDA memory: {torch.cuda.memory_allocated()/1e9:.2f} GB\")"
       ],
-      "execution_count": 7,
+      "execution_count": null,
       "outputs": [
         {
           "output_type": "stream",
           "text": [
-            "Loading Qwen/Qwen2.5-1.5B-Instruct (4-bit quantized)...\n"
-          ]
-        },
-        {
-          "output_type": "error",
-          "ename": "ImportError",
-          "evalue": "Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`",
-          "traceback": [
-            "\u001b[31m---------------------------------------------------------------------------\u001b[39m",
-            "\u001b[31mImportError\u001b[39m                               Traceback (most recent call last)",
-            "\u001b[36mCell\u001b[39m\u001b[36m \u001b[39m\u001b[32mIn[7]\u001b[39m\u001b[32m, line 15\u001b[39m\n\u001b[32m     11\u001b[39m )\n\u001b[32m     12\u001b[39m \n\u001b[32m     13\u001b[39m print(f\"Loading {MODEL_NAME} (4-bit quantized)...\")\n\u001b[32m     14\u001b[39m tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, trust_remote_code=\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[32m---> \u001b[39m\u001b[32m15\u001b[39m model = AutoModelForCausalLM.from_pretrained(\n\u001b[32m     16\u001b[39m     MODEL_NAME, trust_remote_code=\u001b[38;5;28;01mTrue\u001b[39;00m,\n\u001b[32m     17\u001b[39m     quantization_config=bnb_config,\n\u001b[32m     18\u001b[39m     device_map=\u001b[33m\"auto\"\u001b[39m,\n",
-            "\u001b[36mFile \u001b[39m\u001b[32m~/viral-posts-env/.venv/lib/python3.14/site-packages/transformers/models/auto/auto_factory.py:394\u001b[39m, in \u001b[36m_BaseAutoModelClass.from_pretrained\u001b[39m\u001b[34m(cls, pretrained_model_name_or_path, *model_args, **kwargs)\u001b[39m\n\u001b[32m    392\u001b[39m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mhasattr\u001b[39m(parent_config, \u001b[33m\"\u001b[39m\u001b[33mquantization_config\u001b[39m\u001b[33m\"\u001b[39m):\n\u001b[32m    393\u001b[39m             config.quantization_config = parent_config.quantization_config\n\u001b[32m--> \u001b[39m\u001b[32m394\u001b[39m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[30;43mmodel_class\u001b[39;49m\u001b[30;43m.\u001b[39;49m\u001b[30;43mfrom_pretrained\u001b[39;49m\u001b[30;43m(\u001b[39;49m\n\u001b[32m    395\u001b[39m \u001b[30;43m        \u001b[39;49m\u001b[30;43mpretrained_model_name_or_path\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43m*\u001b[39;49m\u001b[30;43mmodel_args\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mconfig\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mconfig\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43m*\u001b[39;49m\u001b[30;43m*\u001b[39;49m\u001b[30;43mhub_kwargs\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43m*\u001b[39;49m\u001b[30;43m*\u001b[39;49m\u001b[30;43mkwargs\u001b[39;49m\n\u001b[32m    396\u001b[39m \u001b[30;43m    \u001b[39;49m\u001b[30;43m)\u001b[39;49m\n\u001b[32m    397\u001b[39m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[32m    398\u001b[39m     \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mUnrecognized configuration class \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mconfig.\u001b[34m__class__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m for this kind of AutoModel: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mcls\u001b[39m.\u001b[34m__name__\u001b[39m\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m.\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[33m\"\u001b[39m\n\u001b[32m    399\u001b[39m     \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mModel type should be one of \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m, \u001b[39m\u001b[33m'\u001b[39m.join(c.\u001b[34m__name__\u001b[39m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mfor\u001b[39;00m\u001b[38;5;250m \u001b[39mc\u001b[38;5;250m \u001b[39m\u001b[38;5;129;01min\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28mcls\u001b[39m._model_mapping)\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m.\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m    400\u001b[39m )\n",
-            "\u001b[36mFile \u001b[39m\u001b[32m~/viral-posts-env/.venv/lib/python3.14/site-packages/transformers/modeling_utils.py:4095\u001b[39m, in \u001b[36mPreTrainedModel.from_pretrained\u001b[39m\u001b[34m(cls, pretrained_model_name_or_path, config, cache_dir, ignore_mismatched_sizes, force_download, local_files_only, token, revision, use_safetensors, weights_only, fusion_config, disable_mmap, *model_args, **kwargs)\u001b[39m\n\u001b[32m   4092\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[33m\"\u001b[39m\u001b[33mexperts_implementation\u001b[39m\u001b[33m\"\u001b[39m \u001b[38;5;129;01min\u001b[39;00m kwargs:\n\u001b[32m   4093\u001b[39m     config._experts_implementation = kwargs.pop(\u001b[33m\"\u001b[39m\u001b[33mexperts_implementation\u001b[39m\u001b[33m\"\u001b[39m)\n\u001b[32m-> \u001b[39m\u001b[32m4095\u001b[39m hf_quantizer, config, device_map = \u001b[30;43mget_hf_quantizer\u001b[39;49m\u001b[30;43m(\u001b[39;49m\n\u001b[32m   4096\u001b[39m \u001b[30;43m    \u001b[39;49m\u001b[30;43mconfig\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mquantization_config\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mdevice_map\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43mweights_only\u001b[39;49m\u001b[30;43m,\u001b[39;49m\u001b[30;43m \u001b[39;49m\u001b[30;43muser_agent\u001b[39;49m\n\u001b[32m   4097\u001b[39m \u001b[30;43m\u001b[39;49m\u001b[30;43m)\u001b[39;49m\n\u001b[32m   4099\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m gguf_file:\n\u001b[32m   4100\u001b[39m     \u001b[38;5;28;01mif\u001b[39;00m hf_quantizer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n",
-            "\u001b[36mFile \u001b[39m\u001b[32m~/viral-posts-env/.venv/lib/python3.14/site-packages/transformers/quantizers/auto.py:342\u001b[39m, in \u001b[36mget_hf_quantizer\u001b[39m\u001b[34m(config, quantization_config, device_map, weights_only, user_agent)\u001b[39m\n\u001b[32m    339\u001b[39m     hf_quantizer = \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[32m    341\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m hf_quantizer \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[32m--> \u001b[39m\u001b[32m342\u001b[39m     \u001b[30;43mhf_quantizer\u001b[39;49m\u001b[30;43m.\u001b[39;49m\u001b[30;43mvalidate_environment\u001b[39;49m\u001b[30;43m(\u001b[39;49m\n\u001b[32m    343\u001b[39m \u001b[30;43m        \u001b[39;49m\u001b[30;43mdevice_map\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mdevice_map\u001b[39;49m\u001b[30;43m,\u001b[39;49m\n\u001b[32m    344\u001b[39m \u001b[30;43m        \u001b[39;49m\u001b[30;43mweights_only\u001b[39;49m\u001b[30;43m=\u001b[39;49m\u001b[30;43mweights_only\u001b[39;49m\u001b[30;43m,\u001b[39;49m\n\u001b[32m    345\u001b[39m \u001b[30;43m    \u001b[39;49m\u001b[30;43m)\u001b[39;49m\n\u001b[32m    346\u001b[39m     device_map = hf_quantizer.update_device_map(device_map)\n\u001b[32m    347\u001b[39m     config = hf_quantizer.update_tp_plan(config)\n",
-            "\u001b[36mFile \u001b[39m\u001b[32m~/viral-posts-env/.venv/lib/python3.14/site-packages/transformers/quantizers/quantizer_bnb_4bit.py:62\u001b[39m, in \u001b[36mBnb4BitHfQuantizer.validate_environment\u001b[39m\u001b[34m(self, *args, **kwargs)\u001b[39m\n\u001b[32m     58\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[32m     59\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mUsing `bitsandbytes` 4-bit quantization requires accelerate: `pip install \u001b[39m\u001b[33m'\u001b[39m\u001b[33maccelerate>=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mACCELERATE_MIN_VERSION\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m'\u001b[39m\u001b[33m`\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m     60\u001b[39m     )\n\u001b[32m     61\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_bitsandbytes_available():\n\u001b[32m---> \u001b[39m\u001b[32m62\u001b[39m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\n\u001b[32m     63\u001b[39m         \u001b[33mf\u001b[39m\u001b[33m\"\u001b[39m\u001b[33mUsing `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mBITSANDBYTES_MIN_VERSION\u001b[38;5;132;01m}\u001b[39;00m\u001b[33m`\u001b[39m\u001b[33m\"\u001b[39m\n\u001b[32m     64\u001b[39m     )\n\u001b[32m     66\u001b[39m \u001b[38;5;28;01mfrom\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[34;01m.\u001b[39;00m\u001b[34;01m.\u001b[39;00m\u001b[34;01mintegrations\u001b[39;00m\u001b[38;5;250m \u001b[39m\u001b[38;5;28;01mimport\u001b[39;00m validate_bnb_backend_availability\n\u001b[32m     68\u001b[39m validate_bnb_backend_availability(raise_exception=\u001b[38;5;28;01mTrue\u001b[39;00m)\n",
-            "\u001b[31mImportError\u001b[39m: Using `bitsandbytes` 4-bit quantization requires bitsandbytes: `pip install -U bitsandbytes>=0.46.1`"
+            "Loading Qwen/Qwen2.5-1.5B-Instruct without 4-bit (bitsandbytes/CUDA unavailable).\n",
+            "  On Colab: run `pip install -U bitsandbytes>=0.46.1` and use a GPU runtime.\n",
+            "  On Mac: use fp16 on MPS or fp32 on CPU.\n"
           ]
         }
       ]

training/train_grpo_smoke.ipynb

@@ -0,0 +1,210 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 4,
+  "metadata": {
+    "kernelspec": {
+      "display_name": "Python 3",
+      "language": "python",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python",
+      "version": "3.10.0"
+    }
+  },
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "# `train_grpo_smoke.ipynb` — syntax & environment smoke test\n",
+        "\n",
+        "Companion to `train_grpo.ipynb`. **Fast** (~1–2 min): checks imports, repo layout, `TASK_HORIZON`, and one short env run.\n",
+        "\n",
+        "Run **all cells top to bottom** in Colab or locally before starting the full training notebook."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# Cell 1: Minimal deps (quoted versions for zsh / shell safety)\n",
+        "!pip install -q pydantic httpx\n",
+        "!pip install -q \"openenv-core[core]>=0.2.2\""
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# Cell 2: Repo path (same logic as main notebook)\n",
+        "import os\n",
+        "import sys\n",
+        "import shutil\n",
+        "import subprocess\n",
+        "from pathlib import Path\n",
+        "\n",
+        "REPO_BRANCH = \"hack1\"\n",
+        "REPO_URL = \"https://github.com/VaibhavKhandare/viral-posts-env.git\"\n",
+        "COLAB_REPO = Path(\"/content/viral-posts-env\")\n",
+        "\n",
+        "\n",
+        "def _is_repo_root(p: Path) -> bool:\n",
+        "    return (p / \"server\" / \"viraltest_environment.py\").is_file() and (p / \"models.py\").is_file()\n",
+        "\n",
+        "\n",
+        "def _find_local_root() -> Path:\n",
+        "    here = Path.cwd().resolve()\n",
+        "    for cand in (here, here.parent, here.parent.parent):\n",
+        "        if _is_repo_root(cand):\n",
+        "            return cand\n",
+        "    raise FileNotFoundError(\n",
+        "        \"Could not find project root. cd into viral-posts-env or use Colab.\"\n",
+        "    )\n",
+        "\n",
+        "\n",
+        "if Path(\"/content\").is_dir():\n",
+        "    if COLAB_REPO.exists():\n",
+        "        shutil.rmtree(COLAB_REPO, ignore_errors=True)\n",
+        "    p = subprocess.run(\n",
+        "        [\"git\", \"clone\", \"--branch\", REPO_BRANCH, \"--depth\", \"1\", REPO_URL, str(COLAB_REPO)],\n",
+        "        capture_output=True,\n",
+        "        text=True,\n",
+        "    )\n",
+        "    if p.returncode != 0:\n",
+        "        raise RuntimeError(f\"git clone failed:\\n{p.stderr}\")\n",
+        "    os.chdir(COLAB_REPO)\n",
+        "    print(\"Mode: Colab\")\n",
+        "else:\n",
+        "    os.chdir(_find_local_root())\n",
+        "    print(\"Mode: local\")\n",
+        "\n",
+        "REPO_DIR = str(Path.cwd().resolve())\n",
+        "if REPO_DIR not in sys.path:\n",
+        "    sys.path.insert(0, REPO_DIR)\n",
+        "print(\"REPO_DIR =\", REPO_DIR)"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# Cell 3: Core imports + TASK_HORIZON check\n",
+        "import os\n",
+        "import sys\n",
+        "from pathlib import Path\n",
+        "\n",
+        "if not Path(\"server/viraltest_environment.py\").is_file():\n",
+        "    for cand in (Path.cwd(), Path.cwd().parent, Path.cwd().parent.parent):\n",
+        "        if (cand / \"server\" / \"viraltest_environment.py\").is_file():\n",
+        "            os.chdir(cand)\n",
+        "            s = str(cand.resolve())\n",
+        "            if s not in sys.path:\n",
+        "                sys.path.insert(0, s)\n",
+        "            print(\"Auto chdir:\", s)\n",
+        "            break\n",
+        "    else:\n",
+        "        raise RuntimeError(\"Run Cell 2 first or open from repo root.\")\n",
+        "\n",
+        "from models import ScheduledAction, ToolCall, ViraltestAction\n",
+        "from server.viraltest_environment import (\n",
+        "    ViraltestEnvironment,\n",
+        "    TAG_POOL,\n",
+        "    TASK_HORIZON,\n",
+        "    TOPIC_CATEGORIES,\n",
+        ")\n",
+        "\n",
+        "assert TASK_HORIZON == 30, f\"Expected TASK_HORIZON=30, got {TASK_HORIZON}\"\n",
+        "print(\"OK: TASK_HORIZON =\", TASK_HORIZON)\n",
+        "print(\"OK: tags =\", len(TAG_POOL), \"niches =\", len(TOPIC_CATEGORIES))"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# Cell 4: One minimal episode (syntax + env wiring)\n",
+        "import random\n",
+        "\n",
+        "_rng = random.Random(42)\n",
+        "\n",
+        "\n",
+        "def plan_minimal(obs_dict, day):\n",
+        "    topics = [t for topics in TOPIC_CATEGORIES.values() for t in topics]\n",
+        "    topic = topics[day % len(topics)]\n",
+        "    tags = [TAG_POOL[i % len(TAG_POOL)] for i in range(day, day + 3)]\n",
+        "    return ViraltestAction(\n",
+        "        scheduled_actions=[\n",
+        "            ScheduledAction(\n",
+        "                hour=12,\n",
+        "                action_type=\"post\",\n",
+        "                content_type=\"carousel\",\n",
+        "                topic=topic,\n",
+        "                tags=tags,\n",
+        "                intent=\"save_bait\",\n",
+        "            )\n",
+        "        ]\n",
+        "    )\n",
+        "\n",
+        "\n",
+        "def run_episode(task, plan_fn, seed=42):\n",
+        "    env = ViraltestEnvironment()\n",
+        "    obs = env.reset(task=task, seed=seed)\n",
+        "    obs_dict = obs.model_dump()\n",
+        "    rewards = []\n",
+        "    for day in range(1, TASK_HORIZON + 1):\n",
+        "        obs = env.step(plan_fn(obs_dict, day))\n",
+        "        obs_dict = obs.model_dump()\n",
+        "        rewards.append(obs.reward or 0.0)\n",
+        "        if obs.done:\n",
+        "            break\n",
+        "    gs = (obs.metadata or {}).get(\"grader_score\", 0.0)\n",
+        "    return {\"steps\": len(rewards), \"total_reward\": sum(rewards), \"grader_score\": gs}\n",
+        "\n",
+        "\n",
+        "r = run_episode(\"monthly_engage\", plan_minimal, seed=42)\n",
+        "print(\"Episode result:\", r)\n",
+        "assert r[\"steps\"] == TASK_HORIZON, f\"Expected {TASK_HORIZON} steps, got {r['steps']}\"\n",
+        "print(\"OK: full monthly episode completed\")"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "metadata": {},
+      "execution_count": null,
+      "outputs": [],
+      "source": [
+        "# Cell 5: Optional ML stack (no model download)\n",
+        "mods = [\n",
+        "    \"torch\",\n",
+        "    \"transformers\",\n",
+        "    \"peft\",\n",
+        "    \"trl\",\n",
+        "    \"datasets\",\n",
+        "    \"accelerate\",\n",
+        "]\n",
+        "for m in mods:\n",
+        "    try:\n",
+        "        __import__(m)\n",
+        "        print(\"OK import:\", m)\n",
+        "    except ImportError as e:\n",
+        "        print(\"MISSING (install in full notebook):\", m, \"—\", e)"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {},
+      "source": [
+        "If all cells pass, open `train_grpo.ipynb` and run the full pipeline."
+      ]
+    }
+  ]
+}