feat: add cleanup utility to purge old bad alphas with fake fields + prevent theme repetition in pipeline"

alpha_factory/cleanup.py

@@ -0,0 +1,104 @@
+"""
+Cleanup utility — purge old alphas that used fake/placeholder fields.
+Run: uv run python -m alpha_factory.cleanup
+
+Also provides batch-level dedup to prevent theme repetition.
+"""
+import duckdb
+from pathlib import Path
+
+DB_PATH = Path("factor_store/alphas.duckdb")
+
+# Fields that don't exist on BRAIN — any alpha using these is invalid
+FAKE_FIELDS = {
+    "book_to_price", "earnings_yield", "roe", "roa", "debt_to_equity",
+    "current_ratio", "returns", "close", "volume", "ts_returns",
+    "high", "low", "bid_ask_spread", "volatility", "ivol", "beta",
+    "hv", "atr", "analyst_rating", "estimate_revision", "target_price",
+    "recommendation", "sentiment", "social_volume", "social_score",
+    "news_sentiment", "iv30", "iv60", "iv90", "iv180", "pcr", "skew",
+    "term_structure", "earnings_surprise", "post_earnings_drift",
+    "guidance", "turnover", "amihud_illiquidity", "revenue_growth",
+    "earnings_growth", "asset_growth", "sales_growth", "open",
+    "vwap", "intraday_range", "dividend_yield", "buyback_yield",
+    "shareholder_yield", "fcf_yield",
+}
+
+
+def cleanup_fake_alphas():
+    """Remove alphas that used placeholder field names."""
+    if not DB_PATH.exists():
+        print("No database found.")
+        return
+
+    conn = duckdb.connect(str(DB_PATH))
+    
+    # Get all alphas
+    rows = conn.execute("SELECT alpha_id, expression, fields_used FROM alphas").fetchall()
+    
+    to_delete = []
+    for alpha_id, expression, fields_used in rows:
+        # Check if expression contains fake fields
+        has_fake = False
+        if fields_used:
+            for f in fields_used:
+                if f in FAKE_FIELDS:
+                    has_fake = True
+                    break
+        if not has_fake and expression:
+            # Also check expression text for bare fake field names
+            expr_lower = expression.lower()
+            for fake in FAKE_FIELDS:
+                if fake in expr_lower and fake not in ("close", "high", "low", "open", "volume"):
+                    # close/high/low/open/volume ARE valid BRAIN fields in pv1
+                    has_fake = True
+                    break
+        
+        if has_fake:
+            to_delete.append(alpha_id)
+    
+    if to_delete:
+        placeholders = ",".join(["?" for _ in to_delete])
+        conn.execute(f"DELETE FROM alphas WHERE alpha_id IN ({placeholders})", to_delete)
+        print(f"Deleted {len(to_delete)} alphas with fake/placeholder fields:")
+        for aid in to_delete:
+            print(f"  - {aid}")
+    else:
+        print("No fake alphas found. Database is clean.")
+    
+    # Show remaining
+    count = conn.execute("SELECT COUNT(*) FROM alphas").fetchone()[0]
+    print(f"\nRemaining alphas in store: {count}")
+    conn.close()
+
+
+def cleanup_quoted_expressions():
+    """Fix any expressions that have quoted field names."""
+    if not DB_PATH.exists():
+        return
+    
+    import re
+    conn = duckdb.connect(str(DB_PATH))
+    rows = conn.execute("SELECT alpha_id, expression FROM alphas WHERE expression LIKE '%''%'").fetchall()
+    
+    fixed = 0
+    for alpha_id, expression in rows:
+        clean = re.sub(r"['\"]([a-z][a-z0-9_]+)['\"]", r"\1", expression)
+        if clean != expression:
+            conn.execute("UPDATE alphas SET expression = ? WHERE alpha_id = ?", [clean, alpha_id])
+            fixed += 1
+    
+    if fixed:
+        print(f"Fixed {fixed} expressions with quoted field names.")
+    else:
+        print("No quoted field names found.")
+    conn.close()
+
+
+if __name__ == "__main__":
+    print("=== Alpha Factory Cleanup ===\n")
+    print("1. Removing alphas with fake/placeholder fields...")
+    cleanup_fake_alphas()
+    print("\n2. Fixing quoted field names in expressions...")
+    cleanup_quoted_expressions()
+    print("\nDone!")