110 lines
3.6 KiB
Python
110 lines
3.6 KiB
Python
"""N-gram ROUGE 점수 (계획서 성능지표 No.7, p.24 수식).
|
|
|
|
계획서 평가수식:
|
|
ROUGE-N = Σ_S∈ref Σ_gram_n Count_match(gram_n) / Σ_S∈ref Σ_gram_n Count(gram_n)
|
|
|
|
즉 레퍼런스 n-gram 기준 재현율(recall) 형태. 본 모듈은 No.7 평가를 위해
|
|
ROUGE-1 / ROUGE-2 / ROUGE-L 을 자체 구현한다 (외부 라이브러리 의존 0).
|
|
|
|
토큰화는 두 가지 지원:
|
|
- "lemma": kiwi 형태소 기본형 (한국어 어미 변화에 강건, 권장)
|
|
- "char" : 공백/문자 기준 단순 토큰 (의존성 없이 동작)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import re
|
|
from collections import Counter
|
|
from dataclasses import dataclass
|
|
|
|
|
|
def _char_tokens(text: str) -> list[str]:
|
|
return re.findall(r"[가-힣A-Za-z0-9]+", text.lower())
|
|
|
|
|
|
def tokenize(text: str, mode: str = "lemma") -> list[str]:
|
|
if mode == "lemma":
|
|
try:
|
|
from app.engine.structural import extract_lemmas
|
|
toks = extract_lemmas(text)
|
|
if toks:
|
|
return toks
|
|
except Exception:
|
|
pass
|
|
return _char_tokens(text)
|
|
|
|
|
|
def _ngrams(tokens: list[str], n: int) -> Counter:
|
|
if len(tokens) < n:
|
|
return Counter()
|
|
return Counter(tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1))
|
|
|
|
|
|
@dataclass
|
|
class RougeScore:
|
|
precision: float
|
|
recall: float
|
|
f1: float
|
|
|
|
def as_dict(self) -> dict[str, float]:
|
|
return {"precision": round(self.precision, 4), "recall": round(self.recall, 4), "f1": round(self.f1, 4)}
|
|
|
|
|
|
def _prf(match: int, sys_total: int, ref_total: int) -> RougeScore:
|
|
precision = match / sys_total if sys_total else 0.0
|
|
recall = match / ref_total if ref_total else 0.0
|
|
f1 = 2 * precision * recall / (precision + recall) if (precision + recall) else 0.0
|
|
return RougeScore(precision, recall, f1)
|
|
|
|
|
|
def rouge_n(system: str, reference: str, n: int = 1, mode: str = "lemma") -> RougeScore:
|
|
sys_g = _ngrams(tokenize(system, mode), n)
|
|
ref_g = _ngrams(tokenize(reference, mode), n)
|
|
match = sum((sys_g & ref_g).values())
|
|
return _prf(match, sum(sys_g.values()), sum(ref_g.values()))
|
|
|
|
|
|
def _lcs_length(a: list[str], b: list[str]) -> int:
|
|
if not a or not b:
|
|
return 0
|
|
prev = [0] * (len(b) + 1)
|
|
for x in a:
|
|
cur = [0] * (len(b) + 1)
|
|
for j, y in enumerate(b, 1):
|
|
cur[j] = prev[j - 1] + 1 if x == y else max(prev[j], cur[j - 1])
|
|
prev = cur
|
|
return prev[-1]
|
|
|
|
|
|
def rouge_l(system: str, reference: str, mode: str = "lemma") -> RougeScore:
|
|
s, r = tokenize(system, mode), tokenize(reference, mode)
|
|
lcs = _lcs_length(s, r)
|
|
return _prf(lcs, len(s), len(r))
|
|
|
|
|
|
def evaluate_pairs(
|
|
pairs: list[tuple[str, str]],
|
|
mode: str = "lemma",
|
|
) -> dict[str, dict[str, float]]:
|
|
"""(system, reference) 페어 리스트 → 코퍼스 평균 ROUGE-1/2/L.
|
|
|
|
계획서 No.2-1년차 목표: N-gram ROUGE 65점 (gpt-4o 줄글 요약 64 대비).
|
|
"""
|
|
if not pairs:
|
|
return {}
|
|
acc = {"rouge1": [], "rouge2": [], "rougeL": []}
|
|
for system, reference in pairs:
|
|
acc["rouge1"].append(rouge_n(system, reference, 1, mode))
|
|
acc["rouge2"].append(rouge_n(system, reference, 2, mode))
|
|
acc["rougeL"].append(rouge_l(system, reference, mode))
|
|
|
|
def avg(scores: list[RougeScore]) -> dict[str, float]:
|
|
k = len(scores)
|
|
return {
|
|
"precision": round(sum(s.precision for s in scores) / k, 4),
|
|
"recall": round(sum(s.recall for s in scores) / k, 4),
|
|
"f1": round(sum(s.f1 for s in scores) / k, 4),
|
|
}
|
|
|
|
return {metric: avg(scores) for metric, scores in acc.items()}
|