tests/test_first_principles.py

#!/usr/bin/env python3
"""
First-Principles Engineering Tests.

T-FP1: State-delta is O(1) — constant tokens regardless of state size
T-FP2: State-delta captures actual changes correctly
T-FP3: Falsification critic generates assertions from code
T-FP4: Falsification score is COMPUTED (0 hallucinations)
T-FP5: Falsification score = passed/total * 10
T-FP6: PEP 578 sandbox policy creation
T-FP7: Path allowlist/blocklist logic correct
"""
import sys, os
sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))

PASS = FAIL = 0
def check(name, cond, detail=""):
    global PASS, FAIL
    PASS += int(cond); FAIL += int(not cond)
    print(f"  {'✓' if cond else '✗'} {name}" + (f": {detail}" if detail and not cond else ""))

from purpose_agent.types import State

# ═══ State Delta (O(1) token cost) ═══
print("═══ State-Delta Critic (Markovian, O(1)) ═══")
from purpose_agent.state_delta import compute_state_delta, format_critic_input, StateDelta

# T-FP1: Constant token cost regardless of state size
big_state = State(data={f"key_{i}": f"value_{i}" * 100 for i in range(100)})  # ~100KB state
big_state2 = State(data={**big_state.data, "new_key": "new_value"})

delta = compute_state_delta(big_state, big_state2)
check("T-FP1 Delta is O(1)", delta.token_estimate < 100,
      f"tokens={delta.token_estimate} (state was ~100KB)")

# Even with 1000-key state, delta is still tiny
huge = State(data={f"k{i}": i for i in range(1000)})
huge2 = State(data={**huge.data, "k500": 999})  # One change
d2 = compute_state_delta(huge, huge2)
check("T-FP1 1000-key state → tiny delta", d2.token_estimate < 50,
      f"tokens={d2.token_estimate}")

# T-FP2: Captures changes correctly
s1 = State(data={"score": 3, "status": "running", "attempts": 1})
s2 = State(data={"score": 7, "status": "running", "attempts": 2, "output": "done"})
d3 = compute_state_delta(s1, s2)
check("T-FP2 Detects added keys", "output" in d3.added_keys)
check("T-FP2 Detects changed keys", "score" in d3.changed_keys)
check("T-FP2 Changed values correct", d3.changed_keys["score"] == (3, 7))
check("T-FP2 Unchanged keys ignored", "status" not in d3.changed_keys)

# Empty delta
s3 = State(data={"x": 1})
d4 = compute_state_delta(s3, s3)
check("T-FP2 No change = empty delta", d4.is_empty)

# Format for critic
formatted = format_critic_input("Write fibonacci", "submit_code", "I wrote the code", d3, max_tokens=300)
check("T-FP2 Formatted output exists", len(formatted) > 0)
check("T-FP2 Under token budget", len(formatted) // 4 <= 300)

# ═══ Falsification Critic (Popperian) ═══
print("\n═══ Falsification Critic (Popper's Method) ═══")
from purpose_agent.falsification_critic import FalsificationCritic, FalsificationResult
from purpose_agent import MockLLMBackend

# T-FP3: Mock LLM generates assertions
mock = MockLLMBackend()
mock.register_handler("TEST ADVERSARY",
    "assert fib(0) == 0\nassert fib(1) == 1\nassert fib(-1) == 0")

critic = FalsificationCritic(llm=mock, timeout_s=5.0)

# Good code — all assertions pass
good_code = "def fib(n):\n    if n <= 0: return 0\n    if n == 1: return 1\n    a,b=0,1\n    for _ in range(2,n+1): a,b=b,a+b\n    return b"
result = critic.evaluate(good_code, purpose="fibonacci")
check("T-FP3 Generates assertions", len(result.generated_assertions) > 0,
      f"got {len(result.generated_assertions)}")
check("T-FP4 Score is computed (not hallucinated)", isinstance(result.score, float))
check("T-FP5 Good code scores high", result.score >= 6.0, f"score={result.score}")
check("T-FP5 Score = passed/total*10",
      abs(result.score - (result.assertions_passed / max(result.assertions_total, 1) * 10)) < 0.1)

# Bad code — assertions fail
bad_code = "def fib(n): return n + 1"  # Wrong implementation
mock2 = MockLLMBackend()
mock2.register_handler("TEST ADVERSARY",
    "assert fib(0) == 0\nassert fib(5) == 5\nassert fib(10) == 55")
critic2 = FalsificationCritic(llm=mock2, timeout_s=5.0)
result2 = critic2.evaluate(bad_code)
check("T-FP5 Bad code scores low", result2.score < 5.0, f"score={result2.score}")
check("T-FP5 Bad code is falsified", result2.is_falsified)

# No code → score 0
result3 = critic.evaluate("")
check("T-FP4 No code → 0", result3.score == 0.0)

# ═══ PEP 578 Sandbox Hooks ═══
print("\n═══ PEP 578 Sandbox (Kernel-Level) ═══")
from purpose_agent.sandbox_hooks import SandboxPolicy, _path_allowed, is_sandbox_installed

# Note: We do NOT actually install the hook in tests (it's permanent + affects test runner)
# Instead we test the LOGIC of the policy

# T-FP6: Policy creation
policy = SandboxPolicy(
    allowed_paths=["/app/workspace", "/tmp"],
    blocked_paths=["/etc", "/proc"],
    block_network=True,
    block_subprocess=True,
)
check("T-FP6 Policy creates", policy is not None)
check("T-FP6 Network blocked", policy.block_network)
check("T-FP6 Subprocess blocked", policy.block_subprocess)
check("T-FP6 Has blocked modules", "ctypes" in policy.blocked_modules)

# T-FP7: Path logic (test without installing hook)
# Monkey-patch the global policy for testing
import purpose_agent.sandbox_hooks as sh
old_policy = sh._policy
sh._policy = policy

check("T-FP7 /tmp allowed", _path_allowed("/tmp/test.py"))
check("T-FP7 /app/workspace allowed", _path_allowed("/app/workspace/code.py"))
check("T-FP7 /etc blocked", not _path_allowed("/etc/passwd"))
check("T-FP7 /proc blocked", not _path_allowed("/proc/self/environ"))

# Restore
sh._policy = old_policy

# Verify not installed in test process
check("T-FP7 Not installed in tests", not is_sandbox_installed())

# ═══ REPORT ═══
print(f"\n{'='*50}")
print(f"  First-Principles Tests: {PASS} pass, {FAIL} fail")
print(f"  {'ALL PASS ✓' if FAIL == 0 else f'{FAIL} FAILURES'}")
print(f"{'='*50}")
sys.exit(0 if FAIL == 0 else 1)