Fix pipeline: enforce ts_decay_linear on all expressions + flip short-horizon returns to reversal"

alpha_factory/personas/expression_compiler.py

@@ -2,6 +2,11 @@
 Expression Compiler — Persona 2 (Hybrid: Jinja + Tinyfish LLM)
 Converts Blueprint JSON → valid BRAIN expression string.
 95% handled by templates. LLM only for novel structures.
+
+POST-COMPILE RULES (mandatory, applied to ALL expressions):
+1. Wrap in ts_decay_linear if decay > 0 (reduces turnover)
+2. Negate if short-horizon returns theme (cross-sectional reversal)
+3. Ensure outer wrapper is unit-safe
 """
 from jinja2 import Environment, BaseLoader
 from ..infra.llm_client import LLMClient
@@ -19,7 +24,8 @@ TEMPLATES = {
 """,
     "multi_horizon_mr": """
 {%- set main = bp.components[0] -%}
-zscore(ts_rank({{ main.fields[0] }}, {{ main.horizon_days }})) * {{ main.weight }} + zscore(ts_rank({{ bp.components[1].fields[0] if bp.components|length > 1 else main.fields[0] }}, {{ bp.components[1].horizon_days if bp.components|length > 1 else 20 }})) * {{ bp.components[1].weight if bp.components|length > 1 else 0.3 }}
+{%- set sign = "-" if bp.components[0].sign_direction == "long_low" else "" -%}
+{{ sign }}(zscore(ts_rank({{ main.fields[0] }}, {{ main.horizon_days }})) * {{ main.weight }}{% for c in bp.components[1:] %} + zscore(ts_rank({{ c.fields[0] }}, {{ c.horizon_days }})) * {{ c.weight }}{% endfor %})
 """,
     "vol_scaled_shock": """
 {%- set c = bp.components[0] -%}
@@ -27,7 +33,7 @@ zscore(ts_delta({{ c.fields[0] }}, {{ c.horizon_days }}) / (ts_std({{ c.fields[0
 """,
     "intraday_mr_decay": """
 {%- set c = bp.components[0] -%}
-ts_decay_linear(zscore(rank({{ c.fields[0] }})), {{ bp.decay }})
+zscore(rank({{ c.fields[0] }}))
 """,
     "pead_revisions": """
 {%- set c = bp.components[0] -%}
@@ -38,7 +44,7 @@ group_zscore(ts_delta({{ c.fields[0] }}, {{ c.horizon_days }}), {{ bp.neutraliza
 {%- for c in bp.components -%}
   {%- set _ = comps.append(c.weight|string ~ " * zscore(rank(" ~ c.fields[0] ~ "))") -%}
 {%- endfor -%}
-ts_decay_linear({{ comps | join(" + ") }}, {{ bp.decay }})
+{{ comps | join(" + ") }}
 """,
 }
 
@@ -53,8 +59,9 @@ RULES:
 2. Every additive operand MUST be wrapped in zscore(), rank(), or group_zscore().
 3. Output ONLY the expression string — no explanation, no markdown.
 4. The expression must be syntactically valid (balanced parentheses, correct arity).
-5. Use ts_decay_linear for smoothing if decay > 0.
-6. Use group_zscore or indneutralize for neutralization within the expression if specified.
+5. Do NOT include ts_decay_linear — that will be added automatically post-compilation.
+6. For short-horizon returns (<=20 days), use NEGATIVE sign (cross-sectional reversal).
+7. Use group_zscore or indneutralize for neutralization within the expression if specified.
 
 Available operators: rank, zscore, group_zscore, group_rank, ts_mean, ts_std, ts_sum,
 ts_delta, ts_decay_linear, ts_rank, ts_argmax, ts_argmin, ts_correlation,
@@ -63,6 +70,40 @@ if_else, less, greater, filter, trade_when, ts_backfill, indneutralize
 """
 
 
+# ─── Post-compilation rules (mandatory) ─────────────────────────────────
+
+def _apply_post_compile_rules(expression: str, blueprint: Blueprint) -> str:
+    """
+    Mandatory post-compilation transformations:
+    1. Apply ts_decay_linear to reduce turnover (ALWAYS, min decay=5)
+    2. Apply sign flip for short-horizon reversal themes
+    3. Ensure the expression won't exceed 70% turnover
+    """
+    expr = expression.strip()
+
+    # Rule 1: Determine if sign should be flipped (short-horizon reversal)
+    # In cross-section, short-term returns (<= 20 days) are mean-reverting
+    needs_flip = False
+    for c in blueprint.components:
+        if any(f in ["returns", "close", "ts_returns"] for f in c.fields):
+            if c.horizon_days <= 20 and c.sign_direction == "long_high":
+                needs_flip = True
+                break
+
+    if needs_flip:
+        expr = f"-({expr})"
+
+    # Rule 2: ALWAYS wrap in ts_decay_linear to control turnover
+    # Minimum decay = 5 days, use blueprint.decay if higher
+    decay = max(blueprint.decay, 5)
+    
+    # Don't double-wrap if already has ts_decay_linear
+    if not expr.startswith("ts_decay_linear("):
+        expr = f"ts_decay_linear({expr}, {decay})"
+
+    return expr
+
+
 async def compile_expression(
     blueprint: Blueprint,
     llm: LLMClient,
@@ -70,7 +111,7 @@ async def compile_expression(
     """
     Convert a Blueprint to a BRAIN expression.
     Uses Jinja template if archetype is known; LLM fallback for novel structures.
-    Uses tier="tinyfish" — ModelManager resolves to user's selected model.
+    ALWAYS applies post-compile rules (decay, sign correction).
     """
     # Try template first (95% of cases)
     if blueprint.archetype in TEMPLATES:
@@ -84,10 +125,13 @@ async def compile_expression(
             fields_used.extend(c.fields)
             ops_used.extend(c.operators)
 
+        # Apply mandatory post-compile rules
+        expr_text = _apply_post_compile_rules(expr_text, blueprint)
+
         return Expression(
             expression=expr_text,
             fields_used=list(set(fields_used)),
-            operators_used=list(set(ops_used)),
+            operators_used=list(set(ops_used)) + ["ts_decay_linear"],
             archetype_used=blueprint.archetype,
         )
 
@@ -99,11 +143,12 @@ Blueprint:
 - Components:
 {_format_components(blueprint)}
 - Neutralization: {blueprint.neutralization.value}
-- Decay: {blueprint.decay}
 - Target: cross-sectional rank → long/short
 
-Output a valid BRAIN expression. Wrap all additive operands in zscore() or rank().
-Apply ts_decay_linear with decay={blueprint.decay} if decay > 0."""
+IMPORTANT: Do NOT include ts_decay_linear — it will be added automatically.
+For short-horizon returns (<=20 days), use NEGATIVE sign (reversal works better cross-sectionally).
+
+Output a valid BRAIN expression. Wrap all additive operands in zscore() or rank()."""
 
     result = await llm.generate_json(
         prompt=user_prompt,
@@ -113,6 +158,11 @@ Apply ts_decay_linear with decay={blueprint.decay} if decay > 0."""
         system_prompt=COMPILER_SYSTEM_PROMPT,
     )
 
+    # Apply mandatory post-compile rules to LLM output too
+    result.expression = _apply_post_compile_rules(result.expression, blueprint)
+    if "ts_decay_linear" not in result.operators_used:
+        result.operators_used.append("ts_decay_linear")
+
     return result