Fix MDF order book depth and CH AI reference price fallback

MDF now places 3 levels of resting bids and asks per symbol each cycle,
ensuring order books always have liquidity. CH AI trader falls back to
securities.txt reference prices when books are empty. CH daily obligation
raised to 20.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

clearing_house/ch_ai_trader.py

@@ -213,10 +213,29 @@ def _run_simulation_cycle():
             time.sleep(0.5)  # stagger submissions
 
 
+def _load_reference_prices() -> dict:
+    """Load reference prices from securities.txt as fallback when books are empty."""
+    ref = {}
+    secs_file = os.getenv("SECURITIES_FILE", "/app/data/securities.txt")
+    try:
+        with open(secs_file) as f:
+            for line in f:
+                line = line.strip()
+                if line and not line.startswith("#"):
+                    parts = line.split()
+                    if len(parts) >= 3:
+                        sym, _, current = parts[0], float(parts[1]), float(parts[2])
+                        ref[sym] = current
+                    elif len(parts) >= 2:
+                        ref[parts[0]] = float(parts[1])
+    except Exception:
+        pass
+    return ref
+
+
 def _fetch_bbos() -> dict:
-    """Get BBO for all symbols from Matcher API."""
+    """Get BBO for all symbols from Matcher API, falling back to reference prices."""
     try:
-        # Load securities list to know symbols
         secs_file = os.getenv("SECURITIES_FILE", "/app/data/securities.txt")
         symbols = []
         try:
@@ -248,6 +267,22 @@ def _fetch_bbos() -> dict:
                         bbos[sym] = {"best_bid": best_bid, "best_ask": best_ask}
             except Exception:
                 pass
+
+        # Fall back to reference prices for symbols with no live BBO
+        if len(bbos) < len(symbols):
+            ref_prices = _load_reference_prices()
+            spread = 0.10
+            for sym in symbols:
+                if sym not in bbos and sym in ref_prices:
+                    mid = ref_prices[sym]
+                    bbos[sym] = {
+                        "best_bid": round(mid - spread, 2),
+                        "best_ask": round(mid + spread, 2),
+                    }
+            if ref_prices:
+                print(f"[CH-AI] Using reference prices for {len(bbos)} symbols "
+                      f"({len(bbos) - len([s for s in bbos if bbos[s].get('best_bid')])} from file)")
+
         return bbos
     except Exception as e:
         print(f"[CH-AI] BBO fetch error: {e}")

clearing_house/ch_database.py

@@ -13,7 +13,7 @@ import os
 CH_DB_PATH = os.getenv("CH_DB_PATH", "/app/data/clearing_house.db")
 CH_MEMBERS = [f"USR{i:02d}" for i in range(1, 11)]
 CH_STARTING_CAPITAL = 100_000.0
-CH_DAILY_OBLIGATION = 10  # minimum securities (qty sum) per trading day
+CH_DAILY_OBLIGATION = 20  # minimum securities (qty sum) per trading day
 
 _local = threading.local()
 

md_feeder/mdf_simulator.py

@@ -131,47 +131,48 @@ if __name__ == "__main__":
 
                 mid = vals["current"]
                 half_spread = 0.10
-
-                side = random.choice(["BUY", "SELL"])
-                rand = random.random()
-
-                if rand < 0.90:
-                    # Passive: place orders away from mid to rest on book
+                tick = Config.TICK_SIZE
+
+                # Always place a resting BID and ASK to maintain book depth
+                for depth_level in range(3):
+                    offset = random.randint(1 + depth_level * 3, 3 + depth_level * 5) * tick
+                    bid_price = round(mid - half_spread - offset, 2)
+                    ask_price = round(mid + half_spread + offset, 2)
+                    bid_qty = random.choice([50, 100, 150, 200])
+                    ask_qty = random.choice([50, 100, 150, 200])
+
+                    bid_order = make_order(sym, "BUY", bid_price, bid_qty)
+                    ask_order = make_order(sym, "SELL", ask_price, ask_qty)
+                    producer.send(Config.ORDERS_TOPIC, bid_order)
+                    producer.send(Config.ORDERS_TOPIC, ask_order)
+                    print(f"[MDF] Depth: {sym} BID {bid_qty}@{bid_price:.2f}  ASK {ask_qty}@{ask_price:.2f}")
+
+                # Occasionally add an aggressive order to generate trades (20%)
+                if random.random() < 0.20:
+                    side = random.choice(["BUY", "SELL"])
                     if side == "BUY":
-                        base = mid - half_spread
-                        offset = random.randint(1, 50) * Config.TICK_SIZE
-                        price = round(base - offset, 2)
+                        price = round(mid + half_spread + random.randint(1, 3) * tick, 2)
                     else:
-                        base = mid + half_spread
-                        offset = random.randint(1, 50) * Config.TICK_SIZE
-                        price = round(base + offset, 2)
-                else:
-                    # Aggressive: cross the spread to create trades
-                    if side == "BUY":
-                        price = round(mid + half_spread + random.randint(1, 5) * Config.TICK_SIZE, 2)
-                    else:
-                        price = round(mid - half_spread - random.randint(1, 5) * Config.TICK_SIZE, 2)
-
-                qty = random.choice([50, 100, 150, 200, 250])
-                order = make_order(sym, side, price, qty)
-                producer.send(Config.ORDERS_TOPIC, order)
-                print(f"[MDF] Order: {order}")
+                        price = round(mid - half_spread - random.randint(1, 3) * tick, 2)
+                    qty = random.choice([50, 100, 150])
+                    aggr_order = make_order(sym, side, price, qty)
+                    producer.send(Config.ORDERS_TOPIC, aggr_order)
+                    print(f"[MDF] Aggr:  {sym} {side} {qty}@{price:.2f}")
 
                 # Simulate small price drift (10% chance, max 2 ticks)
                 if random.random() < 0.10:
-                    drift = random.choice([-2, -1, 1, 2]) * Config.TICK_SIZE
+                    drift = random.choice([-2, -1, 1, 2]) * tick
                     new_price = vals["current"] + drift
                     if new_price >= 1.00:
                         vals["current"] = round(new_price, 2)
                         save_securities(_securities)
 
-                best_bid = mid - half_spread
-                best_ask = mid + half_spread
-                bid_size = random.choice([50, 100, 200])
-                ask_size = random.choice([50, 100, 200])
+                best_bid = round(mid - half_spread, 2)
+                best_ask = round(mid + half_spread, 2)
+                bid_size = random.choice([100, 200, 300])
+                ask_size = random.choice([100, 200, 300])
                 snap = make_snapshot(sym, best_bid, best_ask, bid_size, ask_size)
                 producer.send(Config.SNAPSHOTS_TOPIC, snap)
-                print(f"[MDF] Snapshot: {snap}")
 
                 time.sleep(ORDER_INTERVAL)
 

notebooks/stockex-clearing-house-llm-fine-tuning.ipynb

@@ -1251,7 +1251,7 @@
   },
   {
    "cell_type": "code",
-   "source": "import gc\nfrom peft import PeftModel as PeftModelMerge\n\n# ── Free GPU memory ────────────────────────────────────────────────\nprint(\"Freeing GPU memory for CPU merge...\")\ntry:\n    del trainer\nexcept NameError:\n    pass\ntry:\n    del model\nexcept NameError:\n    pass\ngc.collect()\ntorch.cuda.empty_cache()\n\n# ── Load base model on CPU in float16 (~14GB RAM for 7B) ──────────\nprint(f\"Loading base model on CPU: {BASE_MODEL}\")\nbase_model_cpu = AutoModelForCausalLM.from_pretrained(\n    BASE_MODEL,\n    torch_dtype=torch.float16,\n    device_map=\"cpu\",\n    trust_remote_code=True,\n    low_cpu_mem_usage=True,\n)\n\n# ── Apply LoRA adapter and merge ──────────────────────────────────\nprint(f\"Applying adapter from {OUTPUT_DIR}...\")\nmerged_model = PeftModelMerge.from_pretrained(\n    base_model_cpu,\n    OUTPUT_DIR,\n    torch_dtype=torch.float16,\n    device_map=\"cpu\",\n)\n\nprint(\"Merging adapter into base model...\")\nmerged_model = merged_model.merge_and_unload()\nprint(\"Merge complete.\")\n\n# ── Push full model + tokenizer to HF Hub ─────────────────────────\nprint(f\"Pushing full merged model to {OUTPUT_REPO}...\")\nmerged_model.push_to_hub(\n    OUTPUT_REPO,\n    commit_message=f\"Full merged model: QLoRA fine-tuned {BASE_MODEL}\",\n    token=HF_TOKEN,\n    max_shard_size=\"2GB\",\n)\ntokenizer.push_to_hub(\n    OUTPUT_REPO,\n    commit_message=f\"Tokenizer for {BASE_MODEL}\",\n    token=HF_TOKEN,\n)\nprint(f\"Full model pushed to https://huggingface.co/{OUTPUT_REPO}\")\n\n# ── Cleanup CPU model ─────────────────────────────────────────────\ndel base_model_cpu, merged_model\ngc.collect()\nprint(\"Done — model is now usable via HF Inference Router API.\")",
+   "source": "import gc\nfrom peft import PeftModel as PeftModelMerge\n\nprint(\"Merging adapter into base model on GPU...\")\n\nbase_model_gpu = AutoModelForCausalLM.from_pretrained(\n    BASE_MODEL,\n    torch_dtype=torch.float16,\n    device_map=\"auto\",\n    trust_remote_code=True,\n    low_cpu_mem_usage=True,\n)\n\nmerged_model = PeftModelMerge.from_pretrained(\n    base_model_gpu, OUTPUT_REPO,\n    torch_dtype=torch.float16,\n)\n\nmerged_model = merged_model.merge_and_unload()\nprint(\"Merge complete.\")\n\n# Save locally first, then push (avoids keeping model in memory during upload)\nMERGED_DIR = \"/kaggle/working/merged\"\nprint(f\"Saving merged model to {MERGED_DIR}...\")\nmerged_model.save_pretrained(MERGED_DIR, max_shard_size=\"2GB\")\n\n# Load and save tokenizer (may not be in memory if training cells were skipped)\ntokenizer = AutoTokenizer.from_pretrained(BASE_MODEL, trust_remote_code=True)\ntokenizer.save_pretrained(MERGED_DIR)\n\ndel base_model_gpu, merged_model\ngc.collect()\ntorch.cuda.empty_cache()\n\n# Push from disk\nfrom huggingface_hub import HfApi\napi = HfApi(token=HF_TOKEN)\napi.upload_folder(\n    folder_path=MERGED_DIR,\n    repo_id=OUTPUT_REPO,\n    commit_message=f\"Full merged model: QLoRA fine-tuned {BASE_MODEL}\",\n)\nprint(f\"Full model pushed to https://huggingface.co/{OUTPUT_REPO}\")",
    "metadata": {
     "id": "save-model",
     "trusted": true

shared/config.py

@@ -40,5 +40,5 @@ class Config:
     CH_DB_PATH: str = os.getenv("CH_DB_PATH", "/app/data/clearing_house.db")
     CH_MEMBERS: list = [f"USR{i:02d}" for i in range(1, 11)]
     CH_STARTING_CAPITAL: float = 100_000.0
-    CH_DAILY_OBLIGATION: int = 10
+    CH_DAILY_OBLIGATION: int = 20
     CH_SERVICE_URL: str = os.getenv("CH_SERVICE_URL", "http://localhost:5004")