tests/e2e_api/generate_summary.py

"""SUMMARY.md generátor a tests/e2e_api/results/ JSON-jeiből.

Használat: python tests/e2e_api/generate_summary.py
Output: tests/e2e_api/results/SUMMARY.md
"""

from __future__ import annotations

import json
from pathlib import Path

RESULTS_DIR = Path(__file__).resolve().parent / "results"


def _read_results() -> list[dict]:
    out = []
    for p in sorted(RESULTS_DIR.glob("*.json")):
        try:
            data = json.loads(p.read_text(encoding="utf-8"))
            data["__file__"] = p.name
            out.append(data)
        except Exception:
            continue
    return out


def _format_assertion(a: dict) -> str:
    icon = "OK" if a.get("passed") else "FAIL"
    t = a.get("type", "?")
    if t == "must_contain_keyword":
        return f"  [{icon}] must contain `{a.get('keyword')}`"
    if t == "must_contain_any_of":
        return f"  [{icon}] must contain any of {a.get('keywords')}"
    if t == "must_not_contain":
        return f"  [{icon}] must NOT contain `{a.get('keyword')}`"
    if t == "risk_count_min":
        return f"  [{icon}] risk_count >= {a.get('min')} (actual: {a.get('actual')})"
    if t == "risk_count_max":
        return f"  [{icon}] risk_count <= {a.get('max')} (actual: {a.get('actual')})"
    if t == "severity_max":
        return f"  [{icon}] severity_max <= `{a.get('max_allowed')}` (actual: `{a.get('actual_max')}`)"
    if t == "doc_type":
        return f"  [{icon}] doc_type == `{a.get('expected')}` (actual: `{a.get('actual')}`)"
    if t == "doc_types_set":
        return f"  [{icon}] doc_types == {a.get('expected')} (actual: {a.get('actual')})"
    if t == "doc_types_all":
        return f"  [{icon}] all doc_types == `{a.get('expected_all')}`"
    return f"  [{icon}] {t}"


def _summary_pipeline(data: dict) -> str:
    lines = []
    name = data.get("test_name") or data.get("__file__", "?").replace(".json", "")
    elapsed = data.get("pipeline_seconds", 0)
    n_doc = data.get("document_count", 0)
    n_risk = data.get("risk_count", 0)
    assertions = data.get("paritas_assertions", [])
    n_pass = sum(1 for a in assertions if a.get("passed"))
    n_fail = sum(1 for a in assertions if not a.get("passed"))
    overall = "PASS" if n_fail == 0 else ("PARTIAL" if n_pass > 0 else "FAIL")

    lines.append(f"### `{name}` — {overall}")
    lines.append(f"- Fájlok: {data.get('files', [])}")
    lines.append(f"- Pipeline-idő: {elapsed:.1f}s, doksik: {n_doc}, risks: {n_risk}")
    lines.append(f"- Assertek: {n_pass} OK, {n_fail} FAIL ({len(assertions)} össz)")
    if assertions:
        lines.append("")
        for a in assertions:
            lines.append(_format_assertion(a))
    # Risk-leírások (ha van)
    risks = data.get("risks", [])
    if risks:
        lines.append("")
        lines.append("**Tényleges risk-ek (top 5):**")
        sev_order = {"magas": 0, "kozepes": 1, "alacsony": 2, "info": 3}
        for r in sorted(risks, key=lambda x: sev_order.get(x.get("sulyossag", "info"), 4))[:5]:
            sev = (r.get("sulyossag") or "info").upper()
            tipus = r.get("tipus") or ""
            jog = r.get("jogszabaly") or ""
            jog_str = f" [{jog}]" if jog else ""
            lines.append(f"  - **{sev}** ({tipus}){jog_str}: {r.get('leiras', '')}")
    # Comparison ha van
    comp = data.get("comparison")
    if comp:
        lines.append("")
        lines.append("**Three-way matching:**")
        matches = comp.get("matches", [])
        for m in matches[:5]:
            sev = m.get("severity", "?")
            field = m.get("field", "")
            msg = m.get("message", "")
            lines.append(f"  - **{sev.upper()}** {field}: {msg}")
    # Package insights
    pkg = data.get("package_insights")
    if pkg:
        lines.append("")
        lines.append("**Package insights — exec summary:**")
        lines.append(f"  > {pkg.get('executive_summary', '')[:300]}")
    # DD report
    dd = data.get("dd_report")
    if dd:
        lines.append("")
        lines.append("**DD report — top red flags:**")
        for flag in (dd.get("top_red_flags") or [])[:5]:
            lines.append(f"  - {flag}")
    lines.append("")
    return "\n".join(lines), overall


def _summary_chat(data: dict) -> tuple[str, str]:
    lines = []
    scenario = data.get("scenario", "?")
    elapsed = data.get("elapsed_seconds", 0)
    qa = data.get("qa", [])
    n_pass = 0
    n_fail = 0
    n_error = 0
    for r in qa:
        if r.get("error"):
            n_error += 1
        elif all(a.get("passed") for a in r.get("assertions", [])):
            n_pass += 1
        else:
            n_fail += 1
    overall = "PASS" if n_fail == 0 and n_error == 0 else ("PARTIAL" if n_pass > 0 else "FAIL")

    lines.append(f"### `chat / {scenario}` — {overall}")
    lines.append(f"- Idő: {elapsed:.1f}s, kérdés: {len(qa)} (OK: {n_pass}, FAIL: {n_fail}, ERROR: {n_error})")
    lines.append("")
    for i, r in enumerate(qa, 1):
        if r.get("error"):
            lines.append(f"**Q{i}**: {r.get('q', '')}")
            lines.append(f"  - ERROR: {r.get('error')}")
            continue
        passed = all(a.get("passed") for a in r.get("assertions", []))
        icon = "OK" if passed else "FAIL"
        lines.append(f"**Q{i}** [{icon}]: {r.get('q', '')}")
        ans = (r.get("a") or "").strip().replace("\n", " ")
        lines.append(f"  - A: {ans[:300]}{'...' if len(ans) > 300 else ''}")
        for a in r.get("assertions", []):
            lines.append(_format_assertion(a))
        lines.append("")
    return "\n".join(lines), overall


def main() -> None:
    results = _read_results()
    if not results:
        print("Nincsenek eredmények a results/ mappában")
        return

    summary_lines = [
        "# E2E API Paritás-teszt SUMMARY",
        "",
        f"_{len(results)} JSON eredmény feldolgozva._",
        "",
        "## Kvantitatív összegzés",
        "",
        "| Csoport | Teszt | Eredmény | Risks | Idő (s) |",
        "|---------|-------|----------|-------|---------|",
    ]

    detailed_blocks = []
    counts = {"PASS": 0, "PARTIAL": 0, "FAIL": 0}

    for data in results:
        if data.get("__file__", "").startswith("10_chat_"):
            block, overall = _summary_chat(data)
            counts[overall] = counts.get(overall, 0) + 1
            qa = data.get("qa", [])
            n_pass = sum(1 for r in qa if not r.get("error") and all(a.get("passed") for a in r.get("assertions", [])))
            summary_lines.append(
                f"| 10_chat | {data.get('scenario', '?')} | **{overall}** | "
                f"{n_pass}/{len(qa)} kérdés | {data.get('elapsed_seconds', 0):.1f} |"
            )
            detailed_blocks.append(block)
        else:
            block, overall = _summary_pipeline(data)
            counts[overall] = counts.get(overall, 0) + 1
            test_name = data.get("test_name", "?")
            n_risk = data.get("risk_count", 0)
            elapsed = data.get("pipeline_seconds", 0)
            csoport = test_name.split("_")[0] if test_name else "?"
            summary_lines.append(
                f"| {csoport} | {test_name} | **{overall}** | {n_risk} | {elapsed:.1f} |"
            )
            detailed_blocks.append(block)

    summary_lines.append("")
    summary_lines.append(
        f"**Összesítés:** PASS: {counts.get('PASS', 0)}, "
        f"PARTIAL: {counts.get('PARTIAL', 0)}, FAIL: {counts.get('FAIL', 0)} (összesen: {len(results)})"
    )
    summary_lines.append("")
    summary_lines.append("---")
    summary_lines.append("")
    summary_lines.append("## Részletes eredmények")
    summary_lines.append("")
    summary_lines.extend(detailed_blocks)

    out_path = RESULTS_DIR / "SUMMARY.md"
    out_path.write_text("\n".join(summary_lines), encoding="utf-8")
    print(f"Generálva: {out_path}")
    print(f"PASS: {counts.get('PASS', 0)}, PARTIAL: {counts.get('PARTIAL', 0)}, FAIL: {counts.get('FAIL', 0)}")


if __name__ == "__main__":
    main()