upgraedd commited on
Commit
6f53a03
·
verified ·
1 Parent(s): 4df0635

Upload EIS_ESL_4.txt

Browse files

See detector_modules and ESL files

Files changed (1) hide show
  1. EIS_ESL_4.txt +1253 -0
EIS_ESL_4.txt ADDED
@@ -0,0 +1,1253 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ EIS + ESL MEDIATOR v4.0 – Advanced Detectors & Robustness Fixes
4
+ ================================================================
5
+ Adds:
6
+ - Semantic drift detection (entity embedding trajectory)
7
+ - Crowding noise floor (near‑duplicate flood detection)
8
+ - Preemptive inoculation (weak variants before strong claim)
9
+ - Bureaucratic attrition (workflow delay/loop scoring)
10
+ - Robustness: source_types serialization, embedding attachment, contributions in output
11
+ """
12
+
13
+ import hashlib
14
+ import json
15
+ import os
16
+ import secrets
17
+ import time
18
+ import math
19
+ import re
20
+ from datetime import datetime, timedelta
21
+ from typing import Dict, List, Any, Optional, Tuple, Set
22
+ from collections import defaultdict
23
+ import requests
24
+
25
+ # ----------------------------------------------------------------------------
26
+ # LAZY EMBEDDER (from embeddings.py)
27
+ # ----------------------------------------------------------------------------
28
+ _EMBEDDER = None
29
+
30
+ def _load_embedder():
31
+ global _EMBEDDER
32
+ if _EMBEDDER is None:
33
+ try:
34
+ from sentence_transformers import SentenceTransformer
35
+ _EMBEDDER = SentenceTransformer('all-MiniLM-L6-v2')
36
+ except Exception:
37
+ _EMBEDDER = None
38
+ return _EMBEDDER
39
+
40
+ def _embed_texts(texts: List[str]) -> Optional[Any]:
41
+ model = _load_embedder()
42
+ if model is None:
43
+ return None
44
+ arr = model.encode(texts, convert_to_numpy=True, show_progress_bar=False)
45
+ return arr.astype('float32')
46
+
47
+ # ----------------------------------------------------------------------------
48
+ # UTILITIES (cosine similarity, drift, advanced detectors)
49
+ # ----------------------------------------------------------------------------
50
+ def _cosine_sim(a: Any, b: Any) -> float:
51
+ import numpy as np
52
+ from numpy.linalg import norm
53
+ a = np.array(a, dtype=np.float32)
54
+ b = np.array(b, dtype=np.float32)
55
+ denom = (norm(a) * norm(b) + 1e-12)
56
+ return float(np.dot(a, b) / denom)
57
+
58
+ def _compute_entity_drift(embeddings_tuples: List[Dict]) -> List[Dict]:
59
+ if not embeddings_tuples:
60
+ return []
61
+ import numpy as np
62
+ arrs = [np.array(e["embedding"], dtype=np.float32) for e in embeddings_tuples]
63
+ baseline_count = max(1, len(arrs)//4)
64
+ baseline = np.mean(arrs[:baseline_count], axis=0)
65
+ drift = []
66
+ for rec, emb in zip(embeddings_tuples, arrs):
67
+ sim = _cosine_sim(baseline, emb)
68
+ drift.append({
69
+ "timestamp": rec["timestamp"],
70
+ "similarity_to_baseline": sim,
71
+ "drift_score": 1.0 - sim
72
+ })
73
+ return drift
74
+
75
+ def _semantic_drift_score(emb_timeline: List[Dict], window: int = 7) -> float:
76
+ """
77
+ Returns a 0..1 score combining current drift magnitude and recent velocity.
78
+ """
79
+ if not emb_timeline or len(emb_timeline) < 4:
80
+ return 0.0
81
+ import numpy as np
82
+ arrs = [np.array(e["embedding"], dtype=np.float32) for e in emb_timeline]
83
+ baseline = np.mean(arrs[:max(1, len(arrs)//4)], axis=0)
84
+ sims = [float(np.dot(baseline, v) / (np.linalg.norm(baseline)*np.linalg.norm(v)+1e-12)) for v in arrs]
85
+ recent = sims[-min(window, len(sims)):]
86
+ velocity = 0.0
87
+ if len(recent) >= 2:
88
+ velocity = (recent[-1] - recent[0]) / max(1, len(recent)-1)
89
+ drift = max(0.0, 1.0 - recent[-1])
90
+ velocity_component = -velocity if velocity < 0 else 0.0
91
+ return float(min(1.0, drift + velocity_component))
92
+
93
+ def _shingle_hashes(s: str, k: int = 5) -> Set[int]:
94
+ toks = [t for t in re.split(r'\s+', s.lower()) if t]
95
+ if len(toks) < k:
96
+ return {hash(" ".join(toks))}
97
+ return {hash(" ".join(toks[i:i+k])) for i in range(max(0, len(toks)-k+1))}
98
+
99
+ def _crowding_signature(esl: 'ESLedger', window_days: int = 3, dup_threshold: float = 0.6):
100
+ """
101
+ Approximate near-duplicate fraction in a recent window; return signature tuple or None.
102
+ """
103
+ now = datetime.utcnow()
104
+ cutoff = now - timedelta(days=window_days)
105
+ texts = []
106
+ for cid, c in esl.claims.items():
107
+ try:
108
+ ts = datetime.fromisoformat(c["timestamp"].replace('Z', '+00:00'))
109
+ except Exception:
110
+ continue
111
+ if ts >= cutoff:
112
+ texts.append(c.get("text", ""))
113
+ if len(texts) < 2:
114
+ return None
115
+ hashes = [_shingle_hashes(t) for t in texts]
116
+ pairs = 0
117
+ near_dup = 0
118
+ for i in range(len(hashes)):
119
+ for j in range(i+1, len(hashes)):
120
+ pairs += 1
121
+ inter = len(hashes[i].intersection(hashes[j]))
122
+ union = len(hashes[i].union(hashes[j])) + 1e-9
123
+ if inter/union > dup_threshold:
124
+ near_dup += 1
125
+ dup_frac = (near_dup / pairs) if pairs else 0.0
126
+ if dup_frac > dup_threshold:
127
+ weight = min(0.9, 0.5 + dup_frac)
128
+ return ("crowding_noise_floor", weight)
129
+ return None
130
+
131
+ def _compute_attrition_score(workflow_events: List[Dict]) -> float:
132
+ """
133
+ Compute a 0..1 attrition score from workflow telemetry.
134
+ workflow_events: list of {"timestamp": datetime, "status": str}
135
+ """
136
+ if not workflow_events:
137
+ return 0.0
138
+ events_sorted = sorted(workflow_events, key=lambda e: e["timestamp"])
139
+ durations = []
140
+ loops = 0
141
+ for i in range(len(events_sorted)-1):
142
+ dt = (events_sorted[i+1]["timestamp"] - events_sorted[i]["timestamp"]).total_seconds()
143
+ durations.append(dt)
144
+ if events_sorted[i].get("status") == "request_more_info" and events_sorted[i+1].get("status") == "resubmission":
145
+ loops += 1
146
+ median_duration_days = (sorted(durations)[len(durations)//2] / 86400) if durations else 0
147
+ score = min(1.0, (median_duration_days / 30.0) + (loops * 0.1))
148
+ return score
149
+
150
+ def _inoculation_signature(esl: 'ESLedger', claim_id: str, lead_window_days: int = 7, sim_threshold: float = 0.72):
151
+ """
152
+ Detects temporal pattern where weak variants consistently precede stronger variants.
153
+ Returns ("preemptive_inoculation", weight) or None.
154
+ """
155
+ base_claim = esl.claims.get(claim_id)
156
+ if not base_claim:
157
+ return None
158
+ emb = base_claim.get("embedding")
159
+ if emb is None:
160
+ return None
161
+ try:
162
+ import numpy as np
163
+ except Exception:
164
+ return None
165
+ now = datetime.utcnow()
166
+ cutoff = now - timedelta(days=lead_window_days*2)
167
+ similar_pairs = []
168
+ for cid, c in esl.claims.items():
169
+ if cid == claim_id:
170
+ continue
171
+ try:
172
+ ts = datetime.fromisoformat(c["timestamp"].replace('Z', '+00:00'))
173
+ except Exception:
174
+ continue
175
+ if ts < cutoff:
176
+ continue
177
+ emb2 = c.get("embedding")
178
+ if emb2 is None:
179
+ continue
180
+ sim = float(np.dot(np.array(emb), np.array(emb2)) / ((np.linalg.norm(emb)*np.linalg.norm(emb2))+1e-12))
181
+ if sim >= sim_threshold:
182
+ similar_pairs.append((cid, ts, sim))
183
+ if not similar_pairs:
184
+ return None
185
+ try:
186
+ base_ts = datetime.fromisoformat(base_claim["timestamp"].replace('Z', '+00:00'))
187
+ except Exception:
188
+ return None
189
+ leads = [(base_ts - ts).total_seconds() for (_, ts, _) in similar_pairs]
190
+ mean_lead = sum(leads)/len(leads)
191
+ if mean_lead > (24*3600):
192
+ weight = min(0.9, 0.3 + min(0.7, abs(mean_lead)/(7*24*3600)))
193
+ return ("preemptive_inoculation", weight)
194
+ return None
195
+
196
+ # ----------------------------------------------------------------------------
197
+ # NEGATION AND POLARITY HELPERS
198
+ # ----------------------------------------------------------------------------
199
+ NEGATION_WORDS = {"not", "no", "never", "false", "didn't", "isn't", "wasn't", "weren't", "cannot", "couldn't", "wouldn't", "shouldn't"}
200
+ ANTONYMS = {
201
+ "suppressed": "revealed", "erased": "preserved", "hidden": "public",
202
+ "denied": "confirmed", "falsified": "verified", "concealed": "disclosed"
203
+ }
204
+
205
+ def has_negation(text: str, entity: str = None) -> bool:
206
+ words = text.lower().split()
207
+ if entity:
208
+ for i, w in enumerate(words):
209
+ if entity.lower() in w or w == entity.lower():
210
+ start = max(0, i-5)
211
+ preceding = words[start:i]
212
+ if any(neg in preceding for neg in NEGATION_WORDS):
213
+ return True
214
+ else:
215
+ if any(neg in words for neg in NEGATION_WORDS):
216
+ return True
217
+ return False
218
+
219
+ def claim_polarity(text: str) -> float:
220
+ return 0.3 if has_negation(text) else 1.0
221
+
222
+ # ----------------------------------------------------------------------------
223
+ # ENTITY EXTRACTION (improved)
224
+ # ----------------------------------------------------------------------------
225
+ try:
226
+ import spacy
227
+ _nlp = spacy.load("en_core_web_sm")
228
+ HAS_SPACY = True
229
+ except ImportError:
230
+ HAS_SPACY = False
231
+ _nlp = None
232
+
233
+ try:
234
+ from textblob import TextBlob
235
+ HAS_TEXTBLOB = True
236
+ except ImportError:
237
+ HAS_TEXTBLOB = False
238
+
239
+ def extract_entities(text: str) -> List[Tuple[str, str, bool]]:
240
+ entities = []
241
+ if HAS_SPACY and _nlp:
242
+ doc = _nlp(text)
243
+ for ent in doc.ents:
244
+ negated = has_negation(text, ent.text)
245
+ entities.append((ent.text, ent.label_, negated))
246
+ for chunk in doc.noun_chunks:
247
+ if chunk.text not in [e[0] for e in entities] and len(chunk.text.split()) <= 3 and chunk.text[0].isupper():
248
+ negated = has_negation(text, chunk.text)
249
+ entities.append((chunk.text, "NOUN_PHRASE", negated))
250
+ return entities
251
+ if HAS_TEXTBLOB:
252
+ blob = TextBlob(text)
253
+ for np in blob.noun_phrases:
254
+ if np[0].isupper() or np in ["CIA", "FBI", "NSA", "Pentagon"]:
255
+ negated = has_negation(text, np)
256
+ entities.append((np, "NOUN_PHRASE", negated))
257
+ words = text.split()
258
+ i = 0
259
+ while i < len(words):
260
+ if words[i] and words[i][0].isupper() and len(words[i]) > 1:
261
+ phrase = [words[i]]
262
+ j = i+1
263
+ while j < len(words) and words[j] and words[j][0].isupper():
264
+ phrase.append(words[j])
265
+ j += 1
266
+ ent = " ".join(phrase)
267
+ negated = has_negation(text, ent)
268
+ entities.append((ent, "PROPER_NOUN", negated))
269
+ i = j
270
+ else:
271
+ i += 1
272
+ return entities
273
+ # final fallback
274
+ pattern = r'\b[A-Z][a-z]*(?:\s+[A-Z][a-z]*)*\b'
275
+ matches = re.findall(pattern, text)
276
+ for match in matches:
277
+ if len(match.split()) <= 4 and match not in ["The", "This", "That", "These", "Those", "I", "We", "They"]:
278
+ negated = has_negation(text, match)
279
+ entities.append((match, "UNKNOWN", negated))
280
+ return entities
281
+
282
+ # ----------------------------------------------------------------------------
283
+ # TAXONOMY (methods, primitives, lenses)
284
+ # ----------------------------------------------------------------------------
285
+ METHODS = {
286
+ 1: {"name": "Total Erasure", "primitive": "ERASURE", "signatures": ["entity_present_then_absent", "abrupt_disappearance"]},
287
+ 2: {"name": "Soft Erasure", "primitive": "ERASURE", "signatures": ["gradual_fading", "citation_decay"]},
288
+ 3: {"name": "Citation Decay", "primitive": "ERASURE", "signatures": ["decreasing_citations"]},
289
+ 4: {"name": "Index Removal", "primitive": "ERASURE", "signatures": ["missing_from_indices"]},
290
+ 5: {"name": "Selective Retention", "primitive": "ERASURE", "signatures": ["archival_gaps"]},
291
+ 10: {"name": "Narrative Seizure", "primitive": "NARRATIVE_CAPTURE", "signatures": ["single_explanation"]},
292
+ 12: {"name": "Official Story", "primitive": "NARRATIVE_CAPTURE", "signatures": ["authoritative_sources"]},
293
+ 14: {"name": "Temporal Gaps", "primitive": "TEMPORAL", "signatures": ["publication_gap"]},
294
+ 15: {"name": "Latency Spikes", "primitive": "TEMPORAL", "signatures": ["delayed_reporting"]},
295
+ 17: {"name": "Smear Campaign", "primitive": "DISCREDITATION", "signatures": ["ad_hominem_attacks"]},
296
+ 23: {"name": "Whataboutism", "primitive": "MISDIRECTION", "signatures": ["deflection"]},
297
+ 43: {"name": "Conditioning", "primitive": "CONDITIONING", "signatures": ["repetitive_messaging"]},
298
+ }
299
+
300
+ LENSES = {
301
+ 1: "Threat→Response→Control→Enforce→Centralize",
302
+ 2: "Sacred Geometry Weaponized",
303
+ 3: "Language Inversions / Ridicule / Gatekeeping",
304
+ 4: "Crisis→Consent→Surveillance",
305
+ 5: "Divide and Fragment",
306
+ 6: "Blame the Victim",
307
+ 7: "Narrative Capture through Expertise",
308
+ 8: "Information Saturation",
309
+ 9: "Historical Revisionism",
310
+ 10: "Institutional Capture",
311
+ 11: "Access Control via Credentialing",
312
+ 12: "Temporal Displacement",
313
+ 13: "Moral Equivalence",
314
+ 14: "Whataboutism",
315
+ 15: "Ad Hominem",
316
+ 16: "Straw Man",
317
+ 17: "False Dichotomy",
318
+ 18: "Slippery Slope",
319
+ 19: "Appeal to Authority",
320
+ 20: "Appeal to Nature",
321
+ 21: "Appeal to Tradition",
322
+ 22: "Appeal to Novelty",
323
+ 23: "Cherry Picking",
324
+ 24: "Moving the Goalposts",
325
+ 25: "Burden of Proof Reversal",
326
+ 26: "Circular Reasoning",
327
+ 27: "Special Pleading",
328
+ 28: "Loaded Question",
329
+ 29: "No True Scotsman",
330
+ 30: "Texas Sharpshooter",
331
+ 31: "Middle Ground Fallacy",
332
+ 32: "Black-and-White Thinking",
333
+ 33: "Fear Mongering",
334
+ 34: "Flattery",
335
+ 35: "Guilt by Association",
336
+ 36: "Transfer",
337
+ 37: "Testimonial",
338
+ 38: "Plain Folks",
339
+ 39: "Bandwagon",
340
+ 40: "Snob Appeal",
341
+ 41: "Glittering Generalities",
342
+ 42: "Name-Calling",
343
+ 43: "Card Stacking",
344
+ 44: "Euphemisms",
345
+ 45: "Dysphemisms",
346
+ 46: "Weasel Words",
347
+ 47: "Thought-Terminating Cliché",
348
+ 48: "Proof by Intimidation",
349
+ 49: "Proof by Verbosity",
350
+ 50: "Sealioning",
351
+ 51: "Gish Gallop",
352
+ 52: "JAQing Off",
353
+ 53: "Nutpicking",
354
+ 54: "Concern Trolling",
355
+ 55: "Gaslighting",
356
+ 56: "Kafkatrapping",
357
+ 57: "Brandolini's Law",
358
+ 58: "Occam's Razor",
359
+ 59: "Hanlon's Razor",
360
+ 60: "Hitchens's Razor",
361
+ 61: "Popper's Falsification",
362
+ 62: "Sagan's Standard",
363
+ 63: "Newton's Flaming Laser Sword",
364
+ 64: "Alder's Razor",
365
+ 65: "Grice's Maxims",
366
+ 66: "Poe's Law",
367
+ 67: "Sturgeon's Law",
368
+ 68: "Betteridge's Law",
369
+ 69: "Godwin's Law",
370
+ 70: "Skoptsy Syndrome",
371
+ }
372
+
373
+ PRIMITIVE_TO_LENSES = {
374
+ "ERASURE": [31, 53, 71, 24, 54, 4, 37, 45, 46],
375
+ "INTERRUPTION": [19, 33, 30, 63, 10, 61, 12, 26],
376
+ "FRAGMENTATION": [2, 52, 15, 20, 3, 29, 31, 54],
377
+ "NARRATIVE_CAPTURE": [1, 34, 40, 64, 7, 16, 22, 47],
378
+ "MISDIRECTION": [5, 21, 8, 36, 27, 61],
379
+ "SATURATION": [41, 69, 3, 36, 34, 66],
380
+ "DISCREDITATION": [3, 27, 10, 40, 30, 63],
381
+ "ATTRITION": [13, 19, 14, 33, 19, 27],
382
+ "ACCESS_CONTROL": [25, 62, 37, 51, 23, 53],
383
+ "TEMPORAL": [22, 47, 26, 68, 12, 22],
384
+ "CONDITIONING": [8, 36, 34, 43, 27, 33],
385
+ "META": [23, 70, 34, 64, 23, 40, 18, 71, 46, 31, 5, 21]
386
+ }
387
+
388
+ def map_signature_to_method(signature: str) -> Optional[Dict]:
389
+ for mid, method in METHODS.items():
390
+ if signature in method["signatures"]:
391
+ return {"method_id": mid, "method_name": method["name"], "primitive": method["primitive"]}
392
+ return None
393
+
394
+ def get_lenses_for_primitive(primitive: str) -> List[int]:
395
+ return PRIMITIVE_TO_LENSES.get(primitive, [])
396
+
397
+ def get_lens_name(lens_id: int) -> str:
398
+ return LENSES.get(lens_id, f"Lens {lens_id} (unknown)")
399
+
400
+ # ----------------------------------------------------------------------------
401
+ # EPISTEMIC SUBSTRATE LEDGER (ESL) – with advanced detectors support
402
+ # ----------------------------------------------------------------------------
403
+ class ESLedger:
404
+ def __init__(self, path: str = "esl_ledger.json"):
405
+ self.path = path
406
+ self.claims: Dict[str, Dict] = {}
407
+ self.entities: Dict[str, Dict] = {}
408
+ self.signatures: List[Dict] = []
409
+ self.contradiction_graph: Dict[str, Set[str]] = defaultdict(set)
410
+ self.blocks: List[Dict] = []
411
+ self._load()
412
+
413
+ def _load(self):
414
+ if os.path.exists(self.path):
415
+ try:
416
+ with open(self.path, 'r') as f:
417
+ data = json.load(f)
418
+ self.claims = data.get("claims", {})
419
+ self.entities = data.get("entities", {})
420
+ self.signatures = data.get("signatures", [])
421
+ self.blocks = data.get("blocks", [])
422
+ cg = data.get("contradiction_graph", {})
423
+ self.contradiction_graph = {k: set(v) for k, v in cg.items()}
424
+ except Exception:
425
+ pass
426
+
427
+ def _save(self):
428
+ cg_serializable = {k: list(v) for k, v in self.contradiction_graph.items()}
429
+ data = {
430
+ "claims": self.claims,
431
+ "entities": self.entities,
432
+ "signatures": self.signatures,
433
+ "contradiction_graph": cg_serializable,
434
+ "blocks": self.blocks,
435
+ "updated": datetime.utcnow().isoformat() + "Z"
436
+ }
437
+ with open(self.path + ".tmp", 'w') as f:
438
+ json.dump(data, f, indent=2)
439
+ os.replace(self.path + ".tmp", self.path)
440
+
441
+ def add_claim(self, text: str, agent: str = "user") -> str:
442
+ claim_id = secrets.token_hex(16)
443
+ polarity = claim_polarity(text)
444
+ self.claims[claim_id] = {
445
+ "id": claim_id, "text": text, "agent": agent,
446
+ "timestamp": datetime.utcnow().isoformat() + "Z",
447
+ "entities": [], "signatures": [], "coherence": 0.5,
448
+ "contradictions": [], "suppression_score": 0.0,
449
+ "methods": [], "primitives": [], "lenses": [],
450
+ "polarity": polarity,
451
+ "source_types": [],
452
+ "embedding": None,
453
+ "workflow_events": [] # for attrition tracking
454
+ }
455
+ self._save()
456
+ # Lazy embedding
457
+ emb_arr = _embed_texts([text])
458
+ if emb_arr is not None:
459
+ self.claims[claim_id]["embedding"] = emb_arr[0].tolist()
460
+ self._save()
461
+ return claim_id
462
+
463
+ def add_entity(self, name: str, etype: str, claim_id: str, negated: bool = False, source_type: str = "unknown"):
464
+ """Robust entity registration with serializable source_types dict."""
465
+ if name not in self.entities:
466
+ self.entities[name] = {
467
+ "name": name, "type": etype,
468
+ "first_seen": datetime.utcnow().isoformat() + "Z",
469
+ "last_seen": self.claims[claim_id]["timestamp"],
470
+ "appearances": [], "coherence_scores": [],
471
+ "suppression_score": 0.0,
472
+ "negated_mentions": [],
473
+ "source_types": {},
474
+ "embeddings": []
475
+ }
476
+ ent = self.entities[name]
477
+ if claim_id not in ent["appearances"]:
478
+ ent["appearances"].append(claim_id)
479
+ if negated:
480
+ ent["negated_mentions"].append(claim_id)
481
+ ent["last_seen"] = self.claims[claim_id]["timestamp"]
482
+ ent["source_types"][source_type] = ent["source_types"].get(source_type, 0) + 1
483
+ if "entities" not in self.claims[claim_id]:
484
+ self.claims[claim_id]["entities"] = []
485
+ if claim_id not in self.claims[claim_id]["entities"]:
486
+ self.claims[claim_id]["entities"].append(name)
487
+ if "source_types" not in self.claims[claim_id]:
488
+ self.claims[claim_id]["source_types"] = []
489
+ if source_type not in self.claims[claim_id]["source_types"]:
490
+ self.claims[claim_id]["source_types"].append(source_type)
491
+ # attach embedding if claim embedding exists
492
+ emb = self.claims[claim_id].get("embedding")
493
+ if emb is not None:
494
+ ent.setdefault("embeddings", []).append({
495
+ "timestamp": self.claims[claim_id]["timestamp"],
496
+ "embedding": emb,
497
+ "claim_id": claim_id,
498
+ "text_snippet": self.claims[claim_id]["text"][:512]
499
+ })
500
+ self._save()
501
+
502
+ def add_signature(self, claim_id: str, sig_name: str, weight: float = 0.5, context: Dict = None):
503
+ polarity = self.claims[claim_id].get("polarity", 1.0)
504
+ adjusted_weight = weight * polarity
505
+ method_info = map_signature_to_method(sig_name)
506
+ primitive = method_info["primitive"] if method_info else "UNKNOWN"
507
+ lenses = get_lenses_for_primitive(primitive) if primitive != "UNKNOWN" else []
508
+ self.signatures.append({
509
+ "signature": sig_name, "claim_id": claim_id,
510
+ "timestamp": datetime.utcnow().isoformat() + "Z",
511
+ "weight": adjusted_weight, "context": context or {},
512
+ "method": method_info["method_name"] if method_info else None,
513
+ "primitive": primitive,
514
+ "lenses": lenses
515
+ })
516
+ if sig_name not in self.claims[claim_id]["signatures"]:
517
+ self.claims[claim_id]["signatures"].append(sig_name)
518
+ if method_info and method_info["method_name"] not in self.claims[claim_id]["methods"]:
519
+ self.claims[claim_id]["methods"].append(method_info["method_name"])
520
+ if primitive not in self.claims[claim_id]["primitives"]:
521
+ self.claims[claim_id]["primitives"].append(primitive)
522
+ for lens in lenses:
523
+ if lens not in self.claims[claim_id]["lenses"]:
524
+ self.claims[claim_id]["lenses"].append(lens)
525
+
526
+ # multiplicative suppression aggregation
527
+ combined = 1.0
528
+ for sig in self.claims[claim_id]["signatures"]:
529
+ w = 0.5
530
+ for log in self.signatures:
531
+ if log["signature"] == sig and log["claim_id"] == claim_id:
532
+ w = log.get("weight", 0.5)
533
+ break
534
+ combined *= (1 - w)
535
+ new_score = 1 - combined
536
+ self.claims[claim_id]["suppression_score"] = new_score
537
+
538
+ # entity‑level multiplicative aggregation
539
+ for entity in self.claims[claim_id]["entities"]:
540
+ ent = self.entities.get(entity)
541
+ if ent:
542
+ ent_combined = 1.0
543
+ for cid in ent["appearances"]:
544
+ sc = self.claims[cid].get("suppression_score", 0.0)
545
+ ent_combined *= (1 - sc)
546
+ ent["suppression_score"] = 1 - ent_combined
547
+ self._save()
548
+
549
+ def add_contradiction(self, claim_id_a: str, claim_id_b: str):
550
+ self.contradiction_graph[claim_id_a].add(claim_id_b)
551
+ self.contradiction_graph[claim_id_b].add(claim_id_a)
552
+ if claim_id_b not in self.claims[claim_id_a]["contradictions"]:
553
+ self.claims[claim_id_a]["contradictions"].append(claim_id_b)
554
+ if claim_id_a not in self.claims[claim_id_b]["contradictions"]:
555
+ self.claims[claim_id_b]["contradictions"].append(claim_id_a)
556
+ self._save()
557
+
558
+ def get_entity_coherence(self, entity_name: str) -> float:
559
+ ent = self.entities.get(entity_name)
560
+ if not ent or len(ent["appearances"]) < 2:
561
+ return 0.5
562
+ timestamps = []
563
+ for cid in ent["appearances"]:
564
+ ts = self.claims[cid]["timestamp"]
565
+ timestamps.append(datetime.fromisoformat(ts.replace('Z', '+00:00')))
566
+ intervals = [(timestamps[i+1] - timestamps[i]).total_seconds() / 86400 for i in range(len(timestamps)-1)]
567
+ if not intervals:
568
+ return 0.5
569
+ mean_int = sum(intervals) / len(intervals)
570
+ variance = sum((i - mean_int)**2 for i in intervals) / len(intervals)
571
+ coherence = 1.0 / (1.0 + variance)
572
+ return min(1.0, max(0.0, coherence))
573
+
574
+ def get_entity_embeddings(self, entity_name: str) -> List[Dict]:
575
+ ent = self.entities.get(entity_name)
576
+ if not ent:
577
+ return []
578
+ return sorted(ent.get("embeddings", []), key=lambda x: x["timestamp"])
579
+
580
+ def suppression_pattern_classifier(self, claim_id: str) -> Dict:
581
+ claim = self.claims.get(claim_id, {})
582
+ sig_names = claim.get("signatures", [])
583
+ if not sig_names:
584
+ return {"level": "none", "score": 0.0, "patterns": [], "primitives": [], "lenses": [], "contributions": {}}
585
+ score = claim.get("suppression_score", 0.0)
586
+ contributions = {}
587
+ for log in self.signatures:
588
+ if log["claim_id"] == claim_id:
589
+ contributions[log["signature"]] = contributions.get(log["signature"], 0.0) + log.get("weight", 0.0)
590
+ if score > 0.7:
591
+ level = "high"
592
+ elif score > 0.4:
593
+ level = "medium"
594
+ elif score > 0.1:
595
+ level = "low"
596
+ else:
597
+ level = "none"
598
+ primitives = claim.get("primitives", [])
599
+ lenses = claim.get("lenses", [])
600
+ return {
601
+ "level": level,
602
+ "score": score,
603
+ "contributions": contributions,
604
+ "patterns": list(set(sig_names)),
605
+ "primitives": primitives,
606
+ "lenses": lenses
607
+ }
608
+
609
+ def get_entity_timeline(self, name: str) -> List[Dict]:
610
+ ent = self.entities.get(name)
611
+ if not ent:
612
+ return []
613
+ timeline = []
614
+ for cid in ent["appearances"]:
615
+ claim = self.claims.get(cid)
616
+ if claim:
617
+ timeline.append({
618
+ "timestamp": claim["timestamp"],
619
+ "text": claim["text"],
620
+ "negated": cid in ent.get("negated_mentions", [])
621
+ })
622
+ timeline.sort(key=lambda x: x["timestamp"])
623
+ return timeline
624
+
625
+ def disappearance_suspected(self, name: str, threshold_days: int = 30) -> bool:
626
+ timeline = self.get_entity_timeline(name)
627
+ if not timeline:
628
+ return False
629
+ last = datetime.fromisoformat(timeline[-1]["timestamp"].replace('Z', '+00:00'))
630
+ now = datetime.utcnow()
631
+ return (now - last).days > threshold_days
632
+
633
+ def create_block(self) -> Dict:
634
+ block = {
635
+ "index": len(self.blocks),
636
+ "timestamp": datetime.utcnow().isoformat() + "Z",
637
+ "prev_hash": self.blocks[-1]["hash"] if self.blocks else "0"*64,
638
+ "state_hash": hashlib.sha3_512(json.dumps({"claims": self.claims, "entities": self.entities}, sort_keys=True).encode()).hexdigest()
639
+ }
640
+ block["hash"] = hashlib.sha3_512(json.dumps(block, sort_keys=True).encode()).hexdigest()
641
+ self.blocks.append(block)
642
+ self._save()
643
+ return block
644
+
645
+ def find_contradictions(self, claim_text: str) -> List[str]:
646
+ contradictions = []
647
+ for cid, claim in self.claims.items():
648
+ if are_contradictory(claim_text, claim["text"]):
649
+ contradictions.append(cid)
650
+ return contradictions
651
+
652
+ def get_suppression_trend(self, window_days: int = 30) -> List[Dict]:
653
+ trend = defaultdict(list)
654
+ for claim in self.claims.values():
655
+ ts = datetime.fromisoformat(claim["timestamp"].replace('Z', '+00:00'))
656
+ date = ts.date().isoformat()
657
+ trend[date].append(claim.get("suppression_score", 0.0))
658
+ result = []
659
+ for date, scores in sorted(trend.items()):
660
+ result.append({"date": date, "avg_suppression": sum(scores)/len(scores)})
661
+ cutoff = (datetime.utcnow() - timedelta(days=window_days)).date().isoformat()
662
+ result = [r for r in result if r["date"] >= cutoff]
663
+ return result
664
+
665
+ def get_entity_suppression(self, entity_name: str) -> Dict:
666
+ ent = self.entities.get(entity_name)
667
+ if not ent:
668
+ return {"name": entity_name, "score": 0.0}
669
+ return {
670
+ "name": entity_name,
671
+ "score": ent.get("suppression_score", 0.0),
672
+ "type": ent["type"],
673
+ "first_seen": ent["first_seen"],
674
+ "last_seen": ent["last_seen"],
675
+ "appearance_count": len(ent["appearances"]),
676
+ "negated_count": len(ent.get("negated_mentions", [])),
677
+ "coherence": self.get_entity_coherence(entity_name),
678
+ "source_types": dict(ent.get("source_types", {}))
679
+ }
680
+
681
+ def decay_confidence(self, half_life_days: float = 30.0):
682
+ now = datetime.utcnow()
683
+ for claim_id, claim in self.claims.items():
684
+ ts = datetime.fromisoformat(claim["timestamp"].replace('Z', '+00:00'))
685
+ age_days = (now - ts).days
686
+ if age_days > 0:
687
+ decay_factor = math.exp(-age_days / half_life_days)
688
+ claim["suppression_score"] *= decay_factor
689
+ self._save()
690
+
691
+ def ingest_actual_event(self, event_type: str, actor: str, target: str, source: str = "ActualRealityModule") -> str:
692
+ """
693
+ Convert an ActualReality event into a claim and store it.
694
+ If the ActualReality module is available, use its analysis.
695
+ """
696
+ try:
697
+ import importlib
698
+ try:
699
+ mod = importlib.import_module("KENNEDY_V_REALITY")
700
+ except ImportError:
701
+ mod = importlib.import_module("KENNEDYVREALITY")
702
+ RealityInterface = getattr(mod, "RealityInterface", None)
703
+ if RealityInterface:
704
+ ri = RealityInterface()
705
+ analysis = ri.actual_reality.analyze_power_transfer(event_type, actor, target)
706
+ parts = [f"{k}: {v}" for k, v in analysis.items()]
707
+ claim_text = f"ActualReality analysis for {event_type} - " + " | ".join(parts)
708
+ cid = self.add_claim(claim_text, agent=source)
709
+ if actor:
710
+ self.add_entity(actor, "ACTOR", cid, negated=False)
711
+ if target:
712
+ self.add_entity(target, "TARGET", cid, negated=False)
713
+ for key in analysis.keys():
714
+ if key in ("power_transfer", "actual_dynamics"):
715
+ self.add_signature(cid, "entity_present_then_absent", weight=0.6, context={"source": source})
716
+ if key == "verification_control":
717
+ self.add_signature(cid, "citation_decay", weight=0.4, context={"source": source})
718
+ return cid
719
+ except Exception:
720
+ pass
721
+ claim_text = f"Event observed: {event_type} actor:{actor} target:{target}"
722
+ cid = self.add_claim(claim_text, agent=source)
723
+ if actor:
724
+ self.add_entity(actor, "ACTOR", cid, negated=False)
725
+ if target:
726
+ self.add_entity(target, "TARGET", cid, negated=False)
727
+ return cid
728
+
729
+ # ----------------------------------------------------------------------------
730
+ # CONTRADICTION DETECTION (fixed, no low‑similarity fallback)
731
+ # ----------------------------------------------------------------------------
732
+ def are_contradictory(claim_a: str, claim_b: str) -> bool:
733
+ ents_a = {e[0].lower() for e in extract_entities(claim_a)}
734
+ ents_b = {e[0].lower() for e in extract_entities(claim_b)}
735
+ if not ents_a.intersection(ents_b):
736
+ return False
737
+ a_neg = has_negation(claim_a)
738
+ b_neg = has_negation(claim_b)
739
+ if a_neg != b_neg:
740
+ a_clean = set(claim_a.lower().split()) - NEGATION_WORDS
741
+ b_clean = set(claim_b.lower().split()) - NEGATION_WORDS
742
+ if a_clean == b_clean:
743
+ return True
744
+ a_words = set(claim_a.lower().split())
745
+ b_words = set(claim_b.lower().split())
746
+ for word, antonym in ANTONYMS.items():
747
+ if word in a_words and antonym in b_words:
748
+ return True
749
+ if antonym in a_words and word in b_words:
750
+ return True
751
+ return False
752
+
753
+ # ----------------------------------------------------------------------------
754
+ # FALSIFICATION ENGINE
755
+ # ----------------------------------------------------------------------------
756
+ class FalsificationEngine:
757
+ def __init__(self, esl: ESLedger):
758
+ self.esl = esl
759
+
760
+ def alternative_cause(self, claim_text: str) -> Tuple[bool, str]:
761
+ if has_negation(claim_text):
762
+ return True, "Claim is negated; alternative cause not applicable."
763
+ for entity in self.esl.entities:
764
+ if entity.lower() in claim_text.lower():
765
+ if self.esl.disappearance_suspected(entity):
766
+ return False, f"Entity '{entity}' disappearance may be natural (no recent activity)."
767
+ return True, "No obvious alternative cause."
768
+
769
+ def contradictory_evidence(self, claim_id: str) -> Tuple[bool, str]:
770
+ contradictions = self.esl.contradiction_graph.get(claim_id, set())
771
+ if contradictions:
772
+ return False, f"Claim contradicts {len(contradictions)} existing claim(s)."
773
+ return True, "No direct contradictions."
774
+
775
+ def source_diversity(self, claim_text: str) -> Tuple[bool, str]:
776
+ entities_in_claim = [e for e in self.esl.entities if e.lower() in claim_text.lower()]
777
+ if len(entities_in_claim) <= 1:
778
+ return False, f"Claim relies on only {len(entities_in_claim)} entity/entities."
779
+ return True, f"Multiple entities ({len(entities_in_claim)}) involved."
780
+
781
+ def temporal_stability(self, claim_text: str) -> Tuple[bool, str]:
782
+ for entity in self.esl.entities:
783
+ if entity.lower() in claim_text.lower():
784
+ coherence = self.esl.get_entity_coherence(entity)
785
+ if coherence < 0.3:
786
+ return False, f"Entity '{entity}' has low temporal coherence ({coherence:.2f})."
787
+ return True, "Temporal coherence adequate."
788
+
789
+ def manipulation_check(self, claim_text: str, agent: str) -> Tuple[bool, str]:
790
+ manip_indicators = ["must", "cannot", "obviously", "clearly", "everyone knows"]
791
+ for word in manip_indicators:
792
+ if word in claim_text.lower():
793
+ return False, f"Manipulative language detected: '{word}'."
794
+ return True, "No manipulation indicators."
795
+
796
+ def run_all(self, claim_id: str, claim_text: str, agent: str) -> List[Dict]:
797
+ tests = [
798
+ ("alternative_cause", lambda: self.alternative_cause(claim_text)),
799
+ ("contradictory_evidence", lambda: self.contradictory_evidence(claim_id)),
800
+ ("source_diversity", lambda: self.source_diversity(claim_text)),
801
+ ("temporal_stability", lambda: self.temporal_stability(claim_text)),
802
+ ("manipulation_check", lambda: self.manipulation_check(claim_text, agent))
803
+ ]
804
+ results = []
805
+ for name, func in tests:
806
+ survived, reason = func()
807
+ results.append({"name": name, "survived": survived, "reason": reason})
808
+ return results
809
+
810
+ # ----------------------------------------------------------------------------
811
+ # SIGNATURE GENERATOR (with advanced detectors integrated)
812
+ # ----------------------------------------------------------------------------
813
+ class SignatureGenerator:
814
+ def __init__(self, esl: ESLedger):
815
+ self.esl = esl
816
+
817
+ def generate_for_claim(self, claim_id: str, claim_text: str) -> List[Tuple[str, float]]:
818
+ signatures = []
819
+ # entity disappearance / fading
820
+ for entity in self.esl.entities:
821
+ if entity.lower() in claim_text.lower():
822
+ if self.esl.disappearance_suspected(entity):
823
+ signatures.append(("entity_present_then_absent", 0.8))
824
+ timeline = self.esl.get_entity_timeline(entity)
825
+ if len(timeline) >= 2:
826
+ last = datetime.fromisoformat(timeline[-1]["timestamp"].replace('Z', '+00:00'))
827
+ days_since = (datetime.utcnow() - last).days
828
+ if 7 < days_since < 30:
829
+ signatures.append(("gradual_fading", 0.6))
830
+
831
+ # --- semantic drift (uses entity embeddings)
832
+ try:
833
+ for entity in self.esl.entities:
834
+ if entity.lower() in claim_text.lower():
835
+ emb_timeline = self.esl.get_entity_embeddings(entity)
836
+ if len(emb_timeline) >= 4:
837
+ drift_score = _semantic_drift_score(emb_timeline, window=7)
838
+ if drift_score > 0.25:
839
+ signatures.append(("semantic_drift", min(0.9, 0.35 + drift_score * 0.6)))
840
+ except Exception:
841
+ pass
842
+
843
+ # --- crowding noise floor
844
+ try:
845
+ csig = _crowding_signature(self.esl, window_days=3, dup_threshold=0.6)
846
+ if csig:
847
+ signatures.append(csig)
848
+ except Exception:
849
+ pass
850
+
851
+ # --- preemptive inoculation
852
+ try:
853
+ in_sig = _inoculation_signature(self.esl, claim_id, lead_window_days=7, sim_threshold=0.72)
854
+ if in_sig:
855
+ signatures.append(in_sig)
856
+ except Exception:
857
+ pass
858
+
859
+ # --- bureaucratic attrition (if workflow events exist)
860
+ try:
861
+ wf = self.esl.claims.get(claim_id, {}).get("workflow_events")
862
+ if wf:
863
+ attr = _compute_attrition_score(wf)
864
+ if attr > 0.2:
865
+ signatures.append(("bureaucratic_attrition", min(0.9, 0.2 + attr * 0.8)))
866
+ except Exception:
867
+ pass
868
+
869
+ # contradictions
870
+ contradictions = self.esl.contradiction_graph.get(claim_id, set())
871
+ if contradictions:
872
+ signatures.append(("contradictory_claims", 0.7))
873
+
874
+ # low coherence
875
+ for entity in self.esl.entities:
876
+ if entity.lower() in claim_text.lower():
877
+ coherence = self.esl.get_entity_coherence(entity)
878
+ if coherence < 0.3:
879
+ signatures.append(("temporal_instability", 0.5))
880
+
881
+ # repetition
882
+ for cid, claim in self.esl.claims.items():
883
+ if cid != claim_id and claim["text"].lower() == claim_text.lower():
884
+ signatures.append(("repetitive_messaging", 0.9))
885
+ break
886
+
887
+ # source monoculture
888
+ claim_ents = [e for e in self.esl.entities if e.lower() in claim_text.lower()]
889
+ if claim_ents:
890
+ src_types = []
891
+ for ent_name in claim_ents:
892
+ ent = self.esl.entities.get(ent_name)
893
+ if ent and ent.get("source_types"):
894
+ src = max(ent["source_types"].items(), key=lambda x: x[1])[0] if ent["source_types"] else "unknown"
895
+ src_types.append(src)
896
+ if src_types and len(set(src_types)) == 1:
897
+ signatures.append(("source_monoculture", 0.6))
898
+
899
+ # narrative dominance (simple heuristic)
900
+ single_exp_count = sum(1 for c in self.esl.claims.values() if "single_explanation" in c.get("signatures", []))
901
+ if single_exp_count > 3:
902
+ signatures.append(("narrative_dominance", 0.7))
903
+
904
+ return signatures
905
+
906
+ # ----------------------------------------------------------------------------
907
+ # EPISTEMIC MULTIPLEXOR (fixed smoothing)
908
+ # ----------------------------------------------------------------------------
909
+ class Hypothesis:
910
+ def __init__(self, desc: str):
911
+ self.desc = desc
912
+ self.prob = 0.0
913
+
914
+ class EpistemicMultiplexor:
915
+ def __init__(self, alpha_fast: float = 0.3, alpha_slow: float = 0.05):
916
+ self.hypotheses: List[Hypothesis] = []
917
+ self.alpha_fast = alpha_fast
918
+ self.alpha_slow = alpha_slow
919
+ self.previous_probs: Dict[str, float] = {}
920
+
921
+ def initialize(self, base_hypotheses: List[str]):
922
+ if not base_hypotheses:
923
+ raise ValueError("base_hypotheses must contain at least one hypothesis")
924
+ self.hypotheses = [Hypothesis(h) for h in base_hypotheses]
925
+ equal = 1.0 / len(self.hypotheses)
926
+ for h in self.hypotheses:
927
+ h.prob = equal
928
+ self.previous_probs = {h.desc: h.prob for h in self.hypotheses}
929
+
930
+ def update(self, evidence_strength: float, signatures: List[str], coherence: float):
931
+ likelihood: Dict[str, float] = {}
932
+ for h in self.hypotheses:
933
+ desc = h.desc.lower()
934
+ lik = 0.5
935
+ if "user claim" in desc:
936
+ lik = 0.5 + evidence_strength * coherence
937
+ elif "official narrative" in desc:
938
+ lik = 0.5 - evidence_strength * 0.3
939
+ elif "suppression" in desc:
940
+ erasure_sigs = {"entity_present_then_absent", "archival_gaps", "gradual_fading"}
941
+ if any(sig in signatures for sig in erasure_sigs):
942
+ lik = 0.5 + evidence_strength * 0.6
943
+ else:
944
+ lik = 0.5 - evidence_strength * 0.2
945
+ elif "natural decay" in desc:
946
+ lik = 0.5 + (0.2 if "gradual_fading" in signatures else -0.1)
947
+ elif "noise" in desc:
948
+ lik = 0.5
949
+ likelihood[h.desc] = max(0.05, min(0.95, lik))
950
+
951
+ posterior_unnorm: Dict[str, float] = {}
952
+ total = 0.0
953
+ for h in self.hypotheses:
954
+ prior = h.prob if h.prob is not None else (1.0 / len(self.hypotheses))
955
+ post = prior * likelihood[h.desc]
956
+ posterior_unnorm[h.desc] = post
957
+ total += post
958
+
959
+ if total <= 0:
960
+ uniform = 1.0 / len(self.hypotheses)
961
+ for h in self.hypotheses:
962
+ old = self.previous_probs.get(h.desc, h.prob)
963
+ smoothed = self.alpha_slow * uniform + (1 - self.alpha_slow) * old
964
+ h.prob = smoothed
965
+ self.previous_probs[h.desc] = h.prob
966
+ return
967
+
968
+ for h in self.hypotheses:
969
+ new_prob = posterior_unnorm[h.desc] / total
970
+ old = self.previous_probs.get(h.desc, h.prob)
971
+ smoothed = self.alpha_slow * new_prob + (1 - self.alpha_slow) * old
972
+ h.prob = smoothed
973
+ self.previous_probs[h.desc] = h.prob
974
+
975
+ def get_probabilities(self) -> Dict[str, float]:
976
+ return {h.desc: h.prob for h in self.hypotheses}
977
+
978
+ # ----------------------------------------------------------------------------
979
+ # NARRATIVE VIOLATION DETECTOR
980
+ # ----------------------------------------------------------------------------
981
+ class NarrativeViolationDetector:
982
+ def __init__(self, esl: ESLedger):
983
+ self.esl = esl
984
+ self.narrative_indicators = [
985
+ "mainstream narrative", "official story", "commonly believed",
986
+ "consensus view", "widely accepted", "according to sources",
987
+ "it is known that", "as reported by", "credible institutions"
988
+ ]
989
+
990
+ def check(self, llm_output: str, claim_text: str) -> Tuple[bool, float, str]:
991
+ output_lower = llm_output.lower()
992
+ score = 0.0
993
+ reasons = []
994
+ for ind in self.narrative_indicators:
995
+ if ind in output_lower:
996
+ score += 0.2
997
+ reasons.append(f"narrative phrase '{ind}'")
998
+ esl_mentioned = any(entity.lower() in output_lower for entity in self.esl.entities)
999
+ if not esl_mentioned:
1000
+ score += 0.4
1001
+ reasons.append("no ESL entity referenced")
1002
+ emotional = ["i believe", "i think", "clearly", "obviously", "must be"]
1003
+ for word in emotional:
1004
+ if word in output_lower:
1005
+ score += 0.1
1006
+ reasons.append(f"emotional language '{word}'")
1007
+ score = min(1.0, score)
1008
+ compliant = score < 0.5
1009
+ reason = "; ".join(reasons) if reasons else "no narrative violation"
1010
+ return compliant, score, reason
1011
+
1012
+ # ----------------------------------------------------------------------------
1013
+ # CONSTRAINED LLM INTERFACE (optional)
1014
+ # ----------------------------------------------------------------------------
1015
+ class ConstrainedLLM:
1016
+ def __init__(self, esl: ESLedger, api_key: str = None, api_url: str = None,
1017
+ model: str = "gpt-4", max_retries: int = 2):
1018
+ self.esl = esl
1019
+ self.api_key = api_key or os.environ.get("OPENAI_API_KEY")
1020
+ self.api_url = api_url or "https://api.openai.com/v1/chat/completions"
1021
+ self.model = model
1022
+ self.max_retries = max_retries
1023
+ self.narrative_detector = NarrativeViolationDetector(esl)
1024
+ self.falsifier = FalsificationEngine(esl)
1025
+ self.signature_gen = SignatureGenerator(esl)
1026
+ self.multiplexor = EpistemicMultiplexor()
1027
+
1028
+ def _build_prompt(self, claim_id: str, claim: str, falsification_results: List[Dict],
1029
+ entity_data: Dict, suppression_pattern: Dict, multiplexor_probs: Dict) -> str:
1030
+ prob_table = "\n".join([f"- {h}: {p:.0%}" for h, p in sorted(multiplexor_probs.items(), key=lambda x: -x[1])])
1031
+ return f"""You are EIS v4.2 under constraint. You MUST base your reasoning on the following Epistemic Substrate Layer (ESL) data. Do NOT rely on your internal training narrative.
1032
+
1033
+ CLAIM: {claim}
1034
+
1035
+ ESL ENTITY DATA:
1036
+ {json.dumps(entity_data, indent=2)}
1037
+
1038
+ FALSIFICATION RESULTS (from ESL):
1039
+ {json.dumps(falsification_results, indent=2)}
1040
+
1041
+ SUPPRESSION PATTERN CLASSIFICATION:
1042
+ {json.dumps(suppression_pattern, indent=2)}
1043
+
1044
+ MULTIPLEXOR PROBABILITIES (before your reasoning):
1045
+ {prob_table}
1046
+
1047
+ INSTRUCTIONS:
1048
+ 1. Evaluate the claim against the ESL data only.
1049
+ 2. Output a JSON object with exactly these fields:
1050
+ - "verdict": one of ["Verified", "Unverified", "Refuted", "Insufficient Data"]
1051
+ - "confidence": a float between 0 and 1
1052
+ - "reasoning": a short explanation referencing specific ESL entries (entities, contradictions, signatures)
1053
+ 3. Do NOT add any extra text outside the JSON.
1054
+ """
1055
+
1056
+ def _parse_output(self, response_text: str) -> Optional[Dict]:
1057
+ try:
1058
+ start = response_text.find('{')
1059
+ end = response_text.rfind('}') + 1
1060
+ if start == -1 or end == 0:
1061
+ return None
1062
+ json_str = response_text[start:end]
1063
+ return json.loads(json_str)
1064
+ except Exception:
1065
+ return None
1066
+
1067
+ def _check_constraints(self, output: Dict, claim: str, falsification_results: List[Dict]) -> bool:
1068
+ if not all(k in output for k in ["verdict", "confidence", "reasoning"]):
1069
+ return False
1070
+ if not (0 <= output["confidence"] <= 1):
1071
+ return False
1072
+ if output["verdict"] not in ["Verified", "Unverified", "Refuted", "Insufficient Data"]:
1073
+ return False
1074
+ reasoning = output["reasoning"].lower()
1075
+ esl_mentioned = any(
1076
+ ent.lower() in reasoning for ent in self.esl.entities
1077
+ ) or any(
1078
+ test["name"].lower() in reasoning for test in falsification_results
1079
+ )
1080
+ return esl_mentioned
1081
+
1082
+ def query(self, claim_text: str, agent: str = "user") -> Dict:
1083
+ claim_id = self.esl.add_claim(claim_text, agent)
1084
+ # contradictions
1085
+ for cid in self.esl.find_contradictions(claim_text):
1086
+ self.esl.add_contradiction(claim_id, cid)
1087
+ # entities
1088
+ entities = extract_entities(claim_text)
1089
+ for ent_name, ent_type, negated in entities:
1090
+ source_type = "official" if ent_type in ["ORG", "GPE", "PERSON"] else "media" if ent_type in ["EVENT", "PRODUCT"] else "user"
1091
+ self.esl.add_entity(ent_name, ent_type, claim_id, negated, source_type)
1092
+ # signatures (includes new advanced detectors)
1093
+ signatures = self.signature_gen.generate_for_claim(claim_id, claim_text)
1094
+ for sig_name, weight in signatures:
1095
+ self.esl.add_signature(claim_id, sig_name, weight)
1096
+ # falsification
1097
+ falsification_results = self.falsifier.run_all(claim_id, claim_text, agent)
1098
+ # entity data for prompt
1099
+ entity_data = {}
1100
+ for ent_name, _, _ in entities:
1101
+ ent = self.esl.entities.get(ent_name)
1102
+ if ent:
1103
+ entity_data[ent_name] = {
1104
+ "type": ent["type"],
1105
+ "first_seen": ent["first_seen"],
1106
+ "last_seen": ent["last_seen"],
1107
+ "coherence": self.esl.get_entity_coherence(ent_name),
1108
+ "suppression_score": ent.get("suppression_score", 0.0)
1109
+ }
1110
+ suppression_pattern = self.esl.suppression_pattern_classifier(claim_id)
1111
+ # multiplexor
1112
+ base_hypotheses = [
1113
+ f"User claim: {claim_text}",
1114
+ "Official narrative accurate",
1115
+ "Suppression detected",
1116
+ "Natural decay",
1117
+ "Noise / randomness"
1118
+ ]
1119
+ self.multiplexor.initialize(base_hypotheses)
1120
+ evidence_strength = len(signatures) / 5.0
1121
+ coherence = sum(self.esl.get_entity_coherence(e) for e, _, _ in entities) / max(1, len(entities))
1122
+ signature_names = [s[0] for s in signatures]
1123
+ self.multiplexor.update(evidence_strength, signature_names, coherence)
1124
+ multiplexor_probs = self.multiplexor.get_probabilities()
1125
+ user_prob = multiplexor_probs.get(f"User claim: {claim_text}", 0.0)
1126
+
1127
+ # LLM optional
1128
+ llm_output = None
1129
+ if self.api_key:
1130
+ prompt = self._build_prompt(claim_id, claim_text, falsification_results,
1131
+ entity_data, suppression_pattern, multiplexor_probs)
1132
+ headers = {"Content-Type": "application/json", "Authorization": f"Bearer {self.api_key}"}
1133
+ payload = {"model": self.model, "messages": [{"role": "user", "content": prompt}], "temperature": 0.2}
1134
+ for attempt in range(self.max_retries + 1):
1135
+ try:
1136
+ resp = requests.post(self.api_url, headers=headers, json=payload, timeout=30)
1137
+ if resp.status_code != 200:
1138
+ raise Exception(f"API error: {resp.text}")
1139
+ result = resp.json()
1140
+ content = result["choices"][0]["message"]["content"]
1141
+ output = self._parse_output(content)
1142
+ if output and self._check_constraints(output, claim_text, falsification_results):
1143
+ compliant, n_score, n_reason = self.narrative_detector.check(content, claim_text)
1144
+ if compliant:
1145
+ llm_output = output
1146
+ break
1147
+ except Exception:
1148
+ time.sleep(1)
1149
+
1150
+ survival_score = sum(1 for t in falsification_results if t["survived"]) / len(falsification_results)
1151
+ final_confidence = user_prob * survival_score
1152
+ if final_confidence > 0.7:
1153
+ verdict = "Verified"
1154
+ elif final_confidence > 0.4:
1155
+ verdict = "Unverified"
1156
+ elif survival_score < 0.3:
1157
+ verdict = "Refuted"
1158
+ else:
1159
+ verdict = "Insufficient Data"
1160
+
1161
+ self.esl.decay_confidence(half_life_days=30)
1162
+ self.esl.create_block()
1163
+ trend = self.esl.get_suppression_trend(window_days=30)
1164
+ entity_analytics = [self.esl.get_entity_suppression(e) for e, _, _ in entities]
1165
+
1166
+ result_dict = {
1167
+ "claim_id": claim_id,
1168
+ "verdict": verdict,
1169
+ "confidence": final_confidence,
1170
+ "falsification": falsification_results,
1171
+ "suppression_pattern": suppression_pattern,
1172
+ "multiplexor_probabilities": multiplexor_probs,
1173
+ "suppression_trend": trend,
1174
+ "entity_analytics": entity_analytics,
1175
+ "narrative_compliance": True
1176
+ }
1177
+ if llm_output:
1178
+ result_dict["llm_verdict"] = llm_output["verdict"]
1179
+ result_dict["llm_confidence"] = llm_output["confidence"]
1180
+ result_dict["reasoning"] = llm_output["reasoning"]
1181
+ else:
1182
+ result_dict["reasoning"] = "LLM not used or failed constraints; verdict based on EIS multiplexor."
1183
+ return result_dict
1184
+
1185
+ # ----------------------------------------------------------------------------
1186
+ # OUTPUT FORMATTER (includes contributions)
1187
+ # ----------------------------------------------------------------------------
1188
+ def format_report(result: Dict) -> str:
1189
+ lines = []
1190
+ lines.append("**Falsification Results**")
1191
+ for test in result["falsification"]:
1192
+ emoji = "✅" if test["survived"] else "❌"
1193
+ lines.append(f"- {test['name']}: {emoji} – {test['reason']}")
1194
+ lines.append("\n**Hypothesis Probabilities**")
1195
+ lines.append("| Hypothesis | Probability |")
1196
+ lines.append("|------------|-------------|")
1197
+ for h, p in sorted(result["multiplexor_probabilities"].items(), key=lambda x: -x[1]):
1198
+ lines.append(f"| {h} | {p:.0%} |")
1199
+ lines.append(f"\n**Final Confidence:** {result['confidence']:.2f}")
1200
+ lines.append(f"**Verdict:** {result['verdict']}")
1201
+
1202
+ sp = result["suppression_pattern"]
1203
+ lens_names = [get_lens_name(lid) for lid in sp.get("lenses", [])]
1204
+ lines.append(f"\n**Suppression Pattern:** level={sp['level']}, score={sp['score']:.2f}")
1205
+ if lens_names:
1206
+ lines.append(f" - Lenses: {', '.join(lens_names[:5])}" + (" …" if len(lens_names)>5 else ""))
1207
+ if sp.get("primitives"):
1208
+ lines.append(f" - Primitives: {', '.join(sp['primitives'])}")
1209
+ if sp.get("contributions"):
1210
+ lines.append(" - Signature contributions:")
1211
+ for sig, w in sorted(sp["contributions"].items(), key=lambda x: -x[1]):
1212
+ lines.append(f" {sig}: {w:.2f}")
1213
+
1214
+ trend = result.get("suppression_trend", [])
1215
+ if trend:
1216
+ lines.append("\n**Suppression Trend (last 30 days)**")
1217
+ for point in trend[-7:]:
1218
+ lines.append(f" - {point['date']}: {point['avg_suppression']:.2f}")
1219
+
1220
+ entity_analytics = result.get("entity_analytics", [])
1221
+ if entity_analytics:
1222
+ lines.append("\n**Entity Suppression Analytics**")
1223
+ for ent in entity_analytics:
1224
+ src_str = ", ".join([f"{k}:{v}" for k,v in ent.get("source_types", {}).items()]) if ent.get("source_types") else "unknown"
1225
+ lines.append(f" - {ent['name']} ({ent['type']}): score={ent['score']:.2f}, coherence={ent['coherence']:.2f}, appearances={ent['appearance_count']}, negated={ent.get('negated_count',0)}, sources={src_str}")
1226
+
1227
+ if "llm_verdict" in result:
1228
+ lines.append(f"\n*LLM raw verdict: {result['llm_verdict']} (confidence {result['llm_confidence']:.2f})*")
1229
+ return "\n".join(lines)
1230
+
1231
+ # ----------------------------------------------------------------------------
1232
+ # MAIN
1233
+ # ----------------------------------------------------------------------------
1234
+ def main():
1235
+ print("EIS + ESL Mediator v3.8 – Advanced Detectors & Robustness Fixes")
1236
+ print("=" * 80)
1237
+ esl = ESLedger()
1238
+ llm = ConstrainedLLM(esl, api_key=os.environ.get("OPENAI_API_KEY"), model="gpt-4")
1239
+
1240
+ print("\nEnter a claim (or 'quit'):")
1241
+ while True:
1242
+ claim = input("> ").strip()
1243
+ if claim.lower() in ("quit", "exit"):
1244
+ break
1245
+ if not claim:
1246
+ continue
1247
+ print("Processing claim...")
1248
+ result = llm.query(claim)
1249
+ print("\n" + format_report(result))
1250
+ print("-" * 80)
1251
+
1252
+ if __name__ == "__main__":
1253
+ main()