5. 실습 ③ — 통합 평가 실행

import json, time
from collections import defaultdict
from graph import app
from state import init_state
from eval.generation import eval_faithfulness, eval_correctness

def run_eval(path="data/eval_dev.json"):
    data = json.load(open(path, encoding="utf-8"))
    rows = []

    for d in data:
        t0 = time.time()
        final = app.invoke(init_state(d["question"]),
                           {"recursion_limit": config.RECURSION_LIMIT})
        elapsed = time.time() - t0

        answer = final["answer"]
        docs   = final.get("documents") or []

        # ① 정답 일치
        correct, reason = eval_correctness(
            d["question"], d["gold_answer"], answer)

        # ② 근거 충실성 (답변 가능한 질문만)
        faith = eval_faithfulness(answer, docs) if d["answerable"] else None

        # ③ 부재 질문 처리 (환각 여부)
        abstained = None
        if not d["answerable"]:
            abstained = ("확인할 수 없" in answer
                         or "찾지 못" in answer
                         or "범위" in answer)

        rows.append({
            "id": d["id"], "type": d["type"],
            "difficulty": d["difficulty"],
            "answerable": d["answerable"],
            "correct": correct, "faith": faith,
            "abstained": abstained,
            "grade": final.get("grade"),
            "researches": final.get("researches", 0),
            "elapsed": round(elapsed, 2),
            "reason": reason,
            "answer": answer[:120],
        })

        mark = "✓" if correct else "✗"
        print(f"{mark} {d['id']} [{d['difficulty']:<6}] "
              f"{elapsed:>5.1f}s  {d['question'][:26]}")

    return rows

def summarize(rows):
    n = len(rows)
    # 답변 가능한 것과 부재 질문을 나눠 담기
    ans_rows = []
    abs_rows = []
    for r in rows:
        if r["answerable"]:
            ans_rows.append(r)
        else:
            abs_rows.append(r)
    correct = sum(r["correct"] for r in rows) / n
    # 신뢰도(faith)가 측정된 것만 골라 담기
    faiths = []
    for r in ans_rows:
        if r["faith"] is not None:
            faiths.append(r["faith"])
    faith   = sum(faiths) / len(faiths) if faiths else 0
    abstain = (sum(r["abstained"] for r in abs_rows) / len(abs_rows)
               if abs_rows else 0)
    times   = sorted(r["elapsed"] for r in rows)
    print(f"\n{'='*56}")
    print(f"■ 전체 평가 (n={n})")
    print(f"   Correctness   : {correct*100:5.1f}%")
    print(f"   Faithfulness  : {faith:5.3f}")
    print(f"   Abstention    : {abstain*100:5.1f}%  "
          f"(부재 질문 {len(abs_rows)}건)")
    print(f"   평균 응답     : {sum(times)/n:5.1f}초")
    print(f"   P95 응답      : {times[int(n*0.95)]:5.1f}초")
    return {"correctness": correct, "faithfulness": faith,
            "abstention": abstain}