diff --git a/scripts/visualize_eval.py b/scripts/visualize_eval.py deleted file mode 100644 index 40062c2..0000000 --- a/scripts/visualize_eval.py +++ /dev/null @@ -1,311 +0,0 @@ -"""평가 결과 시각화 리포트 생성. - -사내 plagia_result 데이터셋 1000쌍에 대해: - - 점수 분포 히스토그램 (메타 임베딩 vs Lemma 교집합) - - threshold-F1 곡선 (모델별) - - 모델 비교 막대차트 - - Markdown 한 페이지 리포트 - -사용: - python scripts/visualize_eval.py \ - --data-dir /Users/marineyang/Desktop/work/code/AI_publish_3rdtest/25/plagia_result -""" - -from __future__ import annotations - -import argparse -import json -import logging -import sys -from pathlib import Path - -import matplotlib -matplotlib.use("Agg") -import matplotlib.font_manager as fm -import matplotlib.pyplot as plt -import numpy as np - -ROOT = Path(__file__).resolve().parent.parent -sys.path.insert(0, str(ROOT)) - -from app.engine.structural import extract_lemmas, lemma_overlap_ratio # noqa: E402 - -logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s: %(message)s") -logger = logging.getLogger("visualize") - - -def _setup_korean_font() -> None: - """macOS 한글 표시용 폰트 설정.""" - candidates = [ - "AppleSDGothicNeo-Regular", "Apple SD Gothic Neo", - "NanumGothic", "Nanum Gothic", - "Malgun Gothic", - ] - available = {f.name for f in fm.fontManager.ttflist} - for name in candidates: - if name in available: - plt.rcParams["font.family"] = name - break - plt.rcParams["axes.unicode_minus"] = False - - -def _load_data(data_dir: Path): - pos = json.load((data_dir / "plagiarism_pos_metadata.json").open(encoding="utf-8")) - neg = json.load((data_dir / "plagiarism_neg_metadata.json").open(encoding="utf-8")) - sims_path = sorted(data_dir.glob("all_similarities_*.json"))[-1] - sims = json.load(sims_path.open(encoding="utf-8")) - meta_map: dict[str, float] = {} - for r in sims.get("pos_results", []): - if r.get("cosine_similarity") is not None: - meta_map[r["id"]] = float(r["cosine_similarity"]) - for r in sims.get("neg_results", []): - if r.get("cosine_similarity") is not None: - meta_map[r["id"]] = float(r["cosine_similarity"]) - - rows = [] - for i, item in enumerate(pos, 1): - sid = f"POS{i:03d}" - if sid not in meta_map: - continue - rows.append((sid, True, item["original_text"], item["augmented_text"], meta_map[sid])) - for i, item in enumerate(neg, 1): - sid = f"NEG{i:03d}" - if sid not in meta_map: - continue - rows.append((sid, False, item["original_text"], item["augmented_text"], meta_map[sid])) - return rows - - -def _compute_lemma(rows): - labels, meta, lemma = [], [], [] - for i, (sid, is_p, orig, aug, m) in enumerate(rows, 1): - q_lemmas = extract_lemmas(aug) - r_lemmas = extract_lemmas(orig) - labels.append(1 if is_p else 0) - meta.append(m) - lemma.append(lemma_overlap_ratio(q_lemmas, r_lemmas)) - if i % 200 == 0: - logger.info("lemma %d/%d", i, len(rows)) - return np.array(labels), np.array(meta), np.array(lemma) - - -def _metrics_at(scores: np.ndarray, labels: np.ndarray, t: float) -> dict: - pred = scores >= t - tp = int(((pred == 1) & (labels == 1)).sum()) - fp = int(((pred == 1) & (labels == 0)).sum()) - tn = int(((pred == 0) & (labels == 0)).sum()) - fn = int(((pred == 0) & (labels == 1)).sum()) - p = tp / (tp + fp) if (tp + fp) else 0.0 - r = tp / (tp + fn) if (tp + fn) else 0.0 - f1 = 2 * p * r / (p + r) if (p + r) else 0.0 - return {"threshold": t, "precision": p, "recall": r, "f1": f1, - "tp": tp, "fp": fp, "tn": tn, "fn": fn} - - -def _curve(scores, labels, grid=None): - if grid is None: - grid = np.arange(0.05, 0.99, 0.01) - return [_metrics_at(scores, labels, float(t)) for t in grid] - - -def plot_distributions(labels, meta, lemma, out_path: Path): - fig, axes = plt.subplots(1, 2, figsize=(13, 5)) - bins = np.linspace(0, 1, 41) - - axes[0].hist(meta[labels == 1], bins=bins, alpha=0.6, label="POS (표절)", color="#d62728") - axes[0].hist(meta[labels == 0], bins=bins, alpha=0.6, label="NEG (비표절)", color="#1f77b4") - axes[0].set_title("메타 임베딩 코사인 점수 분포") - axes[0].set_xlabel("score"); axes[0].set_ylabel("count") - axes[0].legend(); axes[0].grid(alpha=0.3) - axes[0].axvline(0.76, color="black", linestyle="--", alpha=0.5, label="best threshold") - - axes[1].hist(lemma[labels == 1], bins=bins, alpha=0.6, label="POS (표절)", color="#d62728") - axes[1].hist(lemma[labels == 0], bins=bins, alpha=0.6, label="NEG (비표절)", color="#1f77b4") - axes[1].set_title("Lemma 교집합 비율 분포 (구조 분석)") - axes[1].set_xlabel("score"); axes[1].set_ylabel("count") - axes[1].legend(); axes[1].grid(alpha=0.3) - axes[1].axvline(0.59, color="black", linestyle="--", alpha=0.5, label="best threshold") - - fig.suptitle("점수 분포 — POS(표절) vs NEG(비표절) 분리도", fontsize=14) - fig.tight_layout() - fig.savefig(out_path, dpi=120, bbox_inches="tight") - plt.close(fig) - - -def plot_threshold_curves(labels, meta, lemma, out_path: Path): - grid = np.arange(0.05, 0.99, 0.01) - meta_curve = _curve(meta, labels, grid) - lemma_curve = _curve(lemma, labels, grid) - hybrid = 0.30 * meta + 0.70 * lemma - hybrid_curve = _curve(hybrid, labels, grid) - - fig, axes = plt.subplots(1, 2, figsize=(14, 5)) - - # F1 curve - axes[0].plot(grid, [m["f1"] for m in meta_curve], label="메타 임베딩 단독", linewidth=2) - axes[0].plot(grid, [m["f1"] for m in lemma_curve], label="Lemma 단독", linewidth=2) - axes[0].plot(grid, [m["f1"] for m in hybrid_curve], label="하이브리드 (α=0.30)", linewidth=2.5, color="green") - axes[0].set_title("Threshold별 F1 점수") - axes[0].set_xlabel("threshold"); axes[0].set_ylabel("F1") - axes[0].set_ylim(0.0, 1.0); axes[0].grid(alpha=0.3); axes[0].legend() - - # Precision-Recall curve - axes[1].plot([m["recall"] for m in meta_curve], [m["precision"] for m in meta_curve], - label="메타 임베딩 단독", linewidth=2) - axes[1].plot([m["recall"] for m in lemma_curve], [m["precision"] for m in lemma_curve], - label="Lemma 단독", linewidth=2) - axes[1].plot([m["recall"] for m in hybrid_curve], [m["precision"] for m in hybrid_curve], - label="하이브리드 (α=0.30)", linewidth=2.5, color="green") - axes[1].set_title("Precision-Recall Curve") - axes[1].set_xlabel("recall"); axes[1].set_ylabel("precision") - axes[1].set_xlim(0.5, 1.0); axes[1].set_ylim(0.5, 1.0) - axes[1].grid(alpha=0.3); axes[1].legend() - - fig.suptitle("모델 성능 곡선", fontsize=14) - fig.tight_layout() - fig.savefig(out_path, dpi=120, bbox_inches="tight") - plt.close(fig) - - -def plot_model_comparison(labels, meta, lemma, out_path: Path): - grid = np.arange(0.05, 0.99, 0.01) - - def best(scores): - rows = _curve(scores, labels, grid) - return max(rows, key=lambda r: r["f1"]) - - best_meta = best(meta) - best_lemma = best(lemma) - best_hybrid = best(0.30 * meta + 0.70 * lemma) - result_json = {"precision": 0.952, "recall": 0.956, "f1": 0.954} - - models = ["기존 result.json", "메타 단독", "Lemma 단독", "하이브리드 α=0.30"] - precisions = [result_json["precision"], best_meta["precision"], best_lemma["precision"], best_hybrid["precision"]] - recalls = [result_json["recall"], best_meta["recall"], best_lemma["recall"], best_hybrid["recall"]] - f1s = [result_json["f1"], best_meta["f1"], best_lemma["f1"], best_hybrid["f1"]] - - x = np.arange(len(models)) - w = 0.27 - fig, ax = plt.subplots(figsize=(11, 5.5)) - ax.bar(x - w, precisions, w, label="Precision", color="#1f77b4") - ax.bar(x, recalls, w, label="Recall", color="#ff7f0e") - ax.bar(x + w, f1s, w, label="F1", color="#2ca02c") - - for i, (p, r, f1) in enumerate(zip(precisions, recalls, f1s)): - ax.text(i - w, p + 0.005, f"{p:.3f}", ha="center", fontsize=8) - ax.text(i, r + 0.005, f"{r:.3f}", ha="center", fontsize=8) - ax.text(i + w, f1 + 0.005, f"{f1:.3f}", ha="center", fontsize=8) - - ax.set_xticks(x); ax.set_xticklabels(models) - ax.set_ylim(0.7, 1.0); ax.set_ylabel("점수") - ax.set_title("모델 성능 비교 (사내 1000쌍 데이터, F1 최적 threshold 기준)") - ax.grid(alpha=0.3, axis="y"); ax.legend() - fig.tight_layout() - fig.savefig(out_path, dpi=120, bbox_inches="tight") - plt.close(fig) - - return best_meta, best_lemma, best_hybrid, result_json - - -def write_markdown_report(out_path: Path, best_meta, best_lemma, best_hybrid, result_json, - n_total, meta_stats, lemma_stats): - md = f"""# 사내 plagia_result 데이터셋 평가 리포트 - -- **데이터셋**: 표절 페어 {n_total // 2}건 + 비표절 페어 {n_total // 2}건 (총 {n_total}쌍) -- **엔진 버전**: o2o-plagiarism-1.2.0-hybrid-openai -- **하이브리드 결합**: `score = α·meta_emb + (1-α)·lemma_overlap` - -## 1. 점수 분포 (POS vs NEG 분리도) - -| 점수 | POS 평균 | NEG 평균 | **분리도** | std(POS / NEG) | -|---|---|---|---|---| -| 메타 임베딩 코사인 | {meta_stats['pos_avg']:.4f} | {meta_stats['neg_avg']:.4f} | **+{meta_stats['pos_avg'] - meta_stats['neg_avg']:.4f}** | {meta_stats['pos_std']:.3f} / {meta_stats['neg_std']:.3f} | -| **Lemma 교집합 비율** | **{lemma_stats['pos_avg']:.4f}** | **{lemma_stats['neg_avg']:.4f}** | **+{lemma_stats['pos_avg'] - lemma_stats['neg_avg']:.4f}** | {lemma_stats['pos_std']:.3f} / {lemma_stats['neg_std']:.3f} | - -→ Lemma의 분리도가 메타보다 약 2.5배 넓음. 표절-비표절을 점수만으로 더 깨끗하게 구분 가능. - -→ 그래프: `reports/01_score_distributions.png` - -## 2. 모델별 최적 성능 (F1 최대화 threshold) - -| 모델 | Precision | Recall | **F1** | Threshold | -|---|---|---|---|---| -| 기존 result.json (전임자 1단계 산출물) | {result_json['precision']:.4f} | {result_json['recall']:.4f} | **{result_json['f1']:.4f}** | 0.78 | -| 메타 임베딩 단독 | {best_meta['precision']:.4f} | {best_meta['recall']:.4f} | {best_meta['f1']:.4f} | {best_meta['threshold']:.2f} | -| **Lemma 단독** (구조 분석) | **{best_lemma['precision']:.4f}** | **{best_lemma['recall']:.4f}** | **{best_lemma['f1']:.4f}** | {best_lemma['threshold']:.2f} | -| **하이브리드 α=0.30** (Recommended) | **{best_hybrid['precision']:.4f}** | **{best_hybrid['recall']:.4f}** | **{best_hybrid['f1']:.4f}** | {best_hybrid['threshold']:.2f} | - -→ 그래프: `reports/02_threshold_curves.png`, `reports/03_model_comparison.png` - -## 3. Confusion Matrix (하이브리드 α=0.30, threshold={best_hybrid['threshold']:.2f}) - -| | 예측: 표절 | 예측: 비표절 | -|---|---|---| -| **실제: 표절** | TP = {best_hybrid['tp']} | FN = {best_hybrid['fn']} | -| **실제: 비표절** | FP = {best_hybrid['fp']} | TN = {best_hybrid['tn']} | - -## 4. 결론 - -1. **전임자 가이드 검증** — "의미 스코어(메타 임베딩) + 구조 스코어(lemma 교집합) → 하이브리드" 구조가 실제 데이터로 입증됨 -2. **Lemma가 핵심 신호** — augmented 케이스가 "어미·조사만 변경" 패턴이 많아 lemma 단독으로도 F1 {best_lemma['f1']:.4f} 달성 -3. **하이브리드가 가장 안정** — 하이브리드 α=0.30에서 recall {best_hybrid['recall']:.4f} (표절을 거의 다 잡음) -4. **권장 운영 임계치** — `SIMILARITY_THRESHOLD={best_hybrid['threshold']:.2f}`, `WEIGHT_TEXT_SIM=0.30`, `WEIGHT_LEMMA_SIM=0.45` -""" - out_path.write_text(md, encoding="utf-8") - - -def main() -> int: - parser = argparse.ArgumentParser() - parser.add_argument("--data-dir", required=True) - parser.add_argument("--out-dir", default=str(ROOT / "reports")) - args = parser.parse_args() - - _setup_korean_font() - data_dir = Path(args.data_dir).expanduser().resolve() - out_dir = Path(args.out_dir).resolve() - out_dir.mkdir(parents=True, exist_ok=True) - - rows = _load_data(data_dir) - logger.info("Loaded %d valid samples", len(rows)) - labels, meta, lemma = _compute_lemma(rows) - - logger.info("Plotting distributions...") - plot_distributions(labels, meta, lemma, out_dir / "01_score_distributions.png") - logger.info("Plotting threshold curves...") - plot_threshold_curves(labels, meta, lemma, out_dir / "02_threshold_curves.png") - logger.info("Plotting model comparison...") - best_meta, best_lemma, best_hybrid, result_json = plot_model_comparison( - labels, meta, lemma, out_dir / "03_model_comparison.png" - ) - - meta_stats = { - "pos_avg": float(meta[labels == 1].mean()), "pos_std": float(meta[labels == 1].std()), - "neg_avg": float(meta[labels == 0].mean()), "neg_std": float(meta[labels == 0].std()), - } - lemma_stats = { - "pos_avg": float(lemma[labels == 1].mean()), "pos_std": float(lemma[labels == 1].std()), - "neg_avg": float(lemma[labels == 0].mean()), "neg_std": float(lemma[labels == 0].std()), - } - - write_markdown_report( - out_dir / "REPORT.md", - best_meta, best_lemma, best_hybrid, result_json, - len(rows), meta_stats, lemma_stats, - ) - print() - print("=" * 60) - print("리포트 생성 완료") - print("=" * 60) - print(f" 📊 {out_dir / '01_score_distributions.png'}") - print(f" 📊 {out_dir / '02_threshold_curves.png'}") - print(f" 📊 {out_dir / '03_model_comparison.png'}") - print(f" 📄 {out_dir / 'REPORT.md'}") - print() - print(" 열어보기:") - print(f" open {out_dir} # Finder") - print(f" open {out_dir / 'REPORT.md'} # 기본 마크다운 뷰어") - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/scripts/visualize_eval_v2.py b/scripts/visualize_eval_v2.py index 689bacf..c7b4e06 100644 --- a/scripts/visualize_eval_v2.py +++ b/scripts/visualize_eval_v2.py @@ -1,6 +1,6 @@ """KoSimCSE + Lemma + 자카드 통합 평가 시각화 (v2.1.0-kosimcse 기준). -기존 visualize_eval.py 가 OpenAI 메타 임베딩만 측정했던 것을 보강: +OpenAI 메타 임베딩만 보던 이전 버전을 보강: - KoSimCSE 본문 임베딩 코사인 (자체 산출물) - Lemma 교집합 비율 (형태소 구조) - 인물/모티프 자카드 (요소) @@ -9,7 +9,7 @@ 사용: python scripts/visualize_eval_v2.py \ - --data-dir /Users/marineyang/Desktop/work/code/AI_publish_3rdtest/25/plagia_result + --data-dir ./data/eval/plagia_result """ from __future__ import annotations