feat: expression_compiler v2 — new archetypes for model77 SUE/PEAD, supply chain, analyst guidance, PCR contrarian + auto field-aware backfill/sign"

alpha_factory/personas/expression_compiler.py

@@ -1,20 +1,29 @@
 """
-Expression Compiler — Persona 2 (Hybrid: Jinja + Tinyfish LLM)
+Expression Compiler v2 — Persona 2 (Hybrid: Jinja + Tinyfish LLM)
 Converts Blueprint JSON → valid BRAIN expression string.
-95% handled by templates. LLM only for novel structures.
+
+NEW ARCHETYPES (orthogonal to existing 18 alphas):
+- sue_drift: Standardized Unexpected Earnings momentum
+- supply_chain_lead_lag: Customer/competitor return propagation
+- analyst_guidance_yield: Management guidance as yield signal
+- pcr_contrarian: Put-call ratio contrarian reversal
+- model_score_momentum: Multi-factor score derivative momentum
 
 POST-COMPILE RULES (mandatory, applied to ALL expressions):
 1. Wrap in ts_decay_linear if decay > 0 (reduces turnover)
 2. Negate if short-horizon returns theme (cross-sectional reversal)
-3. Ensure outer wrapper is unit-safe
+3. Apply correct sign based on field metadata
 """
 from jinja2 import Environment, BaseLoader
 from ..infra.llm_client import LLMClient
 from ..schemas import Blueprint, Expression
+from ..data.brain_fields import FIELD_INDEX, get_backfill_days, get_sign_multiplier
+from ..data.brain_groups import get_group_for_expression
 
 
-# ─── Jinja Templates for proven archetypes ───────────────────────────────
+# ─── Jinja Templates ─────────────────────────────────────────────────────
 TEMPLATES = {
+    # === EXISTING PROVEN ARCHETYPES ===
     "value_quality_blend": """
 {%- set comps = [] -%}
 {%- for c in bp.components -%}
@@ -45,79 +54,130 @@ group_zscore(ts_delta({{ c.fields[0] }}, {{ c.horizon_days }}), {{ bp.neutraliza
   {%- set _ = comps.append(c.weight|string ~ " * zscore(rank(" ~ c.fields[0] ~ "))") -%}
 {%- endfor -%}
 {{ comps | join(" + ") }}
+""",
+
+    # === NEW ORTHOGONAL ARCHETYPES (model77, analyst, supply chain, options) ===
+
+    "sue_drift": """
+{%- set c = bp.components[0] -%}
+{%- set bf = 10 if c.fields[0] in high_cov else 30 -%}
+group_neutralize(rank(ts_backfill({{ c.fields[0] }}, {{ bf }})), {{ bp.neutralization.value }})
+""",
+
+    "supply_chain_lead_lag": """
+{%- set c = bp.components[0] -%}
+group_neutralize(rank(ts_mean(ts_backfill({{ c.fields[0] }}, 30), {{ c.horizon_days }})), {{ bp.neutralization.value }})
+""",
+
+    "analyst_guidance_yield": """
+{%- set c = bp.components[0] -%}
+{%- set bf = 10 if c.fields[0] in high_cov else 30 -%}
+group_neutralize(zscore(ts_rank(ts_backfill({{ c.fields[0] }}, {{ bf }}), 252)), {{ bp.neutralization.value }})
+""",
+
+    "pcr_contrarian": """
+{%- set c = bp.components[0] -%}
+group_neutralize(rank(-ts_delta(ts_backfill({{ c.fields[0] }}, 30), {{ c.horizon_days }})), {{ bp.neutralization.value }})
+""",
+
+    "model_score_momentum": """
+{%- set c = bp.components[0] -%}
+group_neutralize(zscore(ts_delta({{ c.fields[0] }}, {{ c.horizon_days }})), {{ bp.neutralization.value }})
+""",
+
+    # Alpha 15 archetype (the benchmark) — plug any field into the value leg
+    "alpha15_hybrid": """
+{%- set c = bp.components[0] -%}
+{%- set bf = 10 if c.fields[0] in high_cov else 30 -%}
+{%- set sign_mult = "+" if c.sign_direction == "long_high" else "-" -%}
+intraday_mr = (high + low) / 2 - close;
+leg_raw = ts_backfill({{ c.fields[0] }}, {{ bf }});
+mr_z = zscore(ts_rank(intraday_mr, 252));
+leg_z = {{ sign_mult }}1 * zscore(ts_rank(leg_raw, 252));
+score = 0.60 * mr_z + 0.40 * leg_z;
+group_neutralize(rank(score), {{ bp.neutralization.value }})
 """,
 }
 
 _env = Environment(loader=BaseLoader())
 
+# High-coverage fields (don't need long backfill)
+HIGH_COV_FIELDS = {f.id for f in FIELD_INDEX.values() if f.coverage >= 0.85} if FIELD_INDEX else set()
+
 
 COMPILER_SYSTEM_PROMPT = """You are a BRAIN expression compiler. Your ONLY job is to convert 
 a factor blueprint into a valid WorldQuant BRAIN expression.
 
 RULES:
-1. Use ONLY operators from the BRAIN operator catalogue.
+1. Use ONLY operators from the BRAIN 71-operator catalogue.
 2. Every additive operand MUST be wrapped in zscore(), rank(), or group_zscore().
 3. Output ONLY the expression string — no explanation, no markdown.
 4. The expression must be syntactically valid (balanced parentheses, correct arity).
 5. Do NOT include ts_decay_linear — that will be added automatically post-compilation.
 6. For short-horizon returns (<=20 days), use NEGATIVE sign (cross-sectional reversal).
-7. Use group_zscore or indneutralize for neutralization within the expression if specified.
-
-Available operators: rank, zscore, group_zscore, group_rank, ts_mean, ts_std, ts_sum,
-ts_delta, ts_decay_linear, ts_rank, ts_argmax, ts_argmin, ts_correlation,
-ts_covariance, ts_regression, winsorize, abs, log, sign, power, sqrt, max, min,
-if_else, less, greater, filter, trade_when, ts_backfill, indneutralize
+7. Use group_neutralize for neutralization. Prefer novel group keys over subindustry.
+8. Always ts_backfill fields with coverage < 0.70 using window of 30 days.
+
+Available operators (71 total): rank, zscore, group_zscore, group_rank, group_neutralize,
+indneutralize, ts_mean, ts_std, ts_sum, ts_delta, ts_decay_linear, ts_rank, ts_argmax,
+ts_argmin, ts_correlation, ts_covariance, ts_regression, ts_backfill, ts_delay,
+ts_zscore, ts_skewness, ts_kurtosis, ts_entropy, ts_av_diff, ts_hump, ts_scale,
+ts_decay_exp_window, winsorize, abs, log, sign, power, sqrt, max, min,
+if_else, less, greater, equal, filter, trade_when, mask, vec_avg, vec_sum,
+quantile, bucket, sigmoid, tanh, relu, pasteurize, truncate
 """
 
 
-# ─── Post-compilation rules (mandatory) ─────────────────────────────────
-
 def _apply_post_compile_rules(expression: str, blueprint: Blueprint) -> str:
-    """
-    Mandatory post-compilation transformations:
-    1. Apply ts_decay_linear to reduce turnover (ALWAYS, min decay=5)
-    2. Apply sign flip for short-horizon reversal themes
-    3. Ensure the expression won't exceed 70% turnover
-    """
+    """Mandatory post-compilation transformations."""
     expr = expression.strip()
 
-    # Rule 1: Determine if sign should be flipped (short-horizon reversal)
-    # In cross-section, short-term returns (<= 20 days) are mean-reverting
+    # Rule 1: Sign flip for short-horizon reversal
     needs_flip = False
     for c in blueprint.components:
         if any(f in ["returns", "close", "ts_returns"] for f in c.fields):
             if c.horizon_days <= 20 and c.sign_direction == "long_high":
                 needs_flip = True
                 break
-
     if needs_flip:
         expr = f"-({expr})"
 
-    # Rule 2: ALWAYS wrap in ts_decay_linear to control turnover
-    # Minimum decay = 5 days, use blueprint.decay if higher
+    # Rule 2: Apply field-level sign from registry
+    for c in blueprint.components:
+        for fid in c.fields:
+            if fid in FIELD_INDEX:
+                sign = get_sign_multiplier(FIELD_INDEX[fid])
+                if sign == -1 and c.sign_direction != "long_low":
+                    # Field is inverted but blueprint doesn't know — fix it
+                    if not expr.startswith("-"):
+                        expr = f"-({expr})"
+                    break
+
+    # Rule 3: ALWAYS wrap in ts_decay_linear
     decay = max(blueprint.decay, 5)
-    
-    # Don't double-wrap if already has ts_decay_linear
     if not expr.startswith("ts_decay_linear("):
         expr = f"ts_decay_linear({expr}, {decay})"
 
     return expr
 
 
-async def compile_expression(
-    blueprint: Blueprint,
-    llm: LLMClient,
-) -> Expression:
-    """
-    Convert a Blueprint to a BRAIN expression.
-    Uses Jinja template if archetype is known; LLM fallback for novel structures.
-    ALWAYS applies post-compile rules (decay, sign correction).
-    """
-    # Try template first (95% of cases)
+async def compile_expression(blueprint: Blueprint, llm: LLMClient) -> Expression:
+    """Convert a Blueprint to a BRAIN expression."""
+    
+    # Try template first
     if blueprint.archetype in TEMPLATES:
         template_str = TEMPLATES[blueprint.archetype]
         template = _env.from_string(template_str)
-        expr_text = template.render(bp=blueprint).strip()
+        expr_text = template.render(bp=blueprint, high_cov=HIGH_COV_FIELDS).strip()
+
+        # Clean up multi-line expressions (Alpha15 style)
+        if ";" in expr_text:
+            lines = [l.strip() for l in expr_text.split(";") if l.strip()]
+            expr_text = lines[-1]  # Last line is the final expression
+            # Prepend variable assignments
+            for line in lines[:-1]:
+                var, val = line.split("=", 1)
+                expr_text = expr_text.replace(var.strip(), f"({val.strip()})")
 
         fields_used = []
         ops_used = []
@@ -125,7 +185,6 @@ async def compile_expression(
             fields_used.extend(c.fields)
             ops_used.extend(c.operators)
 
-        # Apply mandatory post-compile rules
         expr_text = _apply_post_compile_rules(expr_text, blueprint)
 
         return Expression(
@@ -145,10 +204,13 @@ Blueprint:
 - Neutralization: {blueprint.neutralization.value}
 - Target: cross-sectional rank → long/short
 
-IMPORTANT: Do NOT include ts_decay_linear — it will be added automatically.
-For short-horizon returns (<=20 days), use NEGATIVE sign (reversal works better cross-sectionally).
+IMPORTANT: 
+- Do NOT include ts_decay_linear (added automatically).
+- For fields with coverage < 0.70, wrap in ts_backfill(field, 30).
+- For short-horizon returns (<=20 days), use NEGATIVE sign.
+- Use group_neutralize with the specified neutralization.
 
-Output a valid BRAIN expression. Wrap all additive operands in zscore() or rank()."""
+Output ONLY the expression string."""
 
     result = await llm.generate_json(
         prompt=user_prompt,
@@ -158,7 +220,6 @@ Output a valid BRAIN expression. Wrap all additive operands in zscore() or rank(
         system_prompt=COMPILER_SYSTEM_PROMPT,
     )
 
-    # Apply mandatory post-compile rules to LLM output too
     result.expression = _apply_post_compile_rules(result.expression, blueprint)
     if "ts_decay_linear" not in result.operators_used:
         result.operators_used.append("ts_decay_linear")