2. 실습 ① — 검색 평가

import json
from nodes.retriever import retriever_node
def eval_retrieval(dataset, k=5):
    rows = []
    for d in dataset:
        if not d["answerable"]:
            continue                       # 부재 질문은 검색 평가 제외
        s = {"question": d["question"], "query": d["question"],
             "top_k": k}
        r = retriever_node(s)
        pages = [doc.metadata["page_no"] for doc in r["documents"]]
        # Hit — 정답 페이지가 하나라도 포함됐나
        hit = False
        for p in pages:
            if p in d["gold_pages"]:
                hit = True
                break
        # Rank — 정답이 몇 번째로 나왔나
        rank = 0
        for i, p in enumerate(pages, 1):
            if p in d["gold_pages"]:
                rank = i
                break
        rows.append({
            "id": d["id"], "type": d["type"],
            "difficulty": d["difficulty"],
            "hit": hit, "rank": rank,
            "rr": 1.0 / rank if rank else 0.0,
            "score": r["best_score"],
            "status": r["retrieval"],
            "found": pages, "gold": d["gold_pages"],
        })
    return rows
def summarize_retrieval(rows, k=5):
    n = len(rows)
    hit_rate = sum(r["hit"] for r in rows) / n
    mrr      = sum(r["rr"] for r in rows) / n
    print(f"■ 검색 평가 (n={n}, k={k})")
    print(f"   Hit Rate@{k} : {hit_rate*100:5.1f}%")
    print(f"   MRR         : {mrr:5.3f}")
    # 1위로 맞힌 개수, 아예 못 찾은 개수 세기
    rank1 = 0
    missed = 0
    for r in rows:
        if r["rank"] == 1:
            rank1 += 1
        if not r["hit"]:
            missed += 1
    print(f"   1위 정답     : {rank1}건")
    print(f"   미검출       : {missed}건")
    return {"hit_rate": hit_rate, "mrr": mrr}
if __name__ == "__main__":
    dev = json.load(open("data/eval_dev.json", encoding="utf-8"))
    rows = eval_retrieval(dev)
    summarize_retrieval(rows)
    print("\n■ 검색 실패 질문")
    for r in rows:
        if not r["hit"]:
            print(f"   {r['id']} [{r['difficulty']}] "
                  f"찾음{r['found']} 정답{r['gold']} "
                  f"점수{r['score']:.2f} ({r['status']})")