o2o-plagiarism-ai/scripts/build_summary_annotation_packet.py

#!/usr/bin/env python3
"""ROUGE 평가용 사람 다중 참조 요약 검수 패킷을 만든다."""

from __future__ import annotations

import argparse
import hashlib
import os
import sys
from collections import defaultdict
from pathlib import Path

if __package__ in (None, ""):
    sys.path.insert(0, str(Path(__file__).resolve().parents[1]))

from app.engine.summarizer import extractive_summary
from app.engine.training_data import (
    pseudonymous_id,
    redact_direct_identifiers,
    sanitize_prompt_metadata,
)


def parse_args() -> argparse.Namespace:
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("xlsx", type=Path)
    parser.add_argument("--out", type=Path, required=True)
    parser.add_argument("--text-column", default="에피소드 본문")
    parser.add_argument("--title-column", default="에피소드 제목")
    parser.add_argument("--group-column", default="id")
    parser.add_argument("--limit", type=int, default=300)
    parser.add_argument("--min-chars", type=int, default=500)
    parser.add_argument("--salt-env", default="DATA_ANONYMIZATION_SALT")
    parser.add_argument("--seed", type=int, default=20260820)
    return parser.parse_args()


def _stable_order(seed: int, key: str) -> str:
    return hashlib.sha256(f"{seed}:{key}".encode()).hexdigest()


def select_group_diverse(rows: list[dict], limit: int, seed: int) -> list[dict]:
    """모든 작성자에서 1건씩 먼저 선택한 후 남은 수를 채운다."""
    grouped: dict[str, list[dict]] = defaultdict(list)
    for row in rows:
        grouped[row["source_group"]].append(row)
    for group_rows in grouped.values():
        group_rows.sort(key=lambda r: _stable_order(seed, r["row_key"]))

    group_names = sorted(grouped, key=lambda g: _stable_order(seed, g))
    chosen: list[dict] = []
    depth = 0
    while len(chosen) < limit:
        added = False
        for group in group_names:
            if depth < len(grouped[group]):
                chosen.append(grouped[group][depth])
                added = True
                if len(chosen) >= limit:
                    break
        if not added:
            break
        depth += 1
    return chosen


def main() -> int:
    args = parse_args()
    from openpyxl import Workbook, load_workbook
    from openpyxl.styles import Alignment, Font, PatternFill

    salt = os.environ.get(args.salt_env, "")
    if not salt:
        print(f"{args.salt_env} 환경변수가 필요합니다.", file=sys.stderr)
        return 2

    source = load_workbook(args.xlsx, read_only=True, data_only=True)
    sheet = source.worksheets[0]
    iterator = sheet.iter_rows(values_only=True)
    headers = [str(v).strip() if v is not None else "" for v in next(iterator)]
    positions = {name: i for i, name in enumerate(headers)}
    missing = [c for c in (args.text_column, args.title_column, args.group_column) if c not in positions]
    if missing:
        print(f"필수 컬럼 없음: {missing}", file=sys.stderr)
        return 2

    candidates = []
    for rownum, values in enumerate(iterator, start=2):
        text = str(values[positions[args.text_column]] or "").strip()
        raw_group = str(values[positions[args.group_column]] or "").strip()
        if len(text) < args.min_chars or not raw_group:
            continue
        text = redact_direct_identifiers(text)
        title = sanitize_prompt_metadata(
            str(values[positions[args.title_column]] or "").strip()
        )
        candidates.append({
            "row_key": f"{sheet.title}:{rownum}",
            "source_group": pseudonymous_id(raw_group, salt),
            "episode_title": title,
            "source_text": text,
        })
    source.close()

    selected = select_group_diverse(candidates, args.limit, args.seed)
    if len(selected) < args.limit:
        print(f"후보가 부족합니다: {len(selected)}/{args.limit}", file=sys.stderr)
        return 2

    workbook = Workbook()
    ws = workbook.active
    ws.title = "annotations"
    columns = [
        "annotation_id", "source_group", "episode_title", "source_text",
        "baseline_extractive", "reference_summary_1", "reviewer_1",
        "reference_summary_2", "reviewer_2", "status", "notes",
    ]
    ws.append(columns)
    for index, row in enumerate(selected, start=1):
        baseline = extractive_summary(row["source_text"], ratio=0.3).final
        ws.append([
            f"summary-{index:04d}", row["source_group"], row["episode_title"],
            row["source_text"], baseline, "", "", "", "", "pending", "",
        ])

    header_fill = PatternFill("solid", fgColor="D9EAF7")
    for cell in ws[1]:
        cell.font = Font(bold=True)
        cell.fill = header_fill
    ws.freeze_panes = "A2"
    ws.auto_filter.ref = ws.dimensions
    widths = {"A": 16, "B": 34, "C": 30, "D": 90, "E": 70,
              "F": 70, "G": 18, "H": 70, "I": 18, "J": 14, "K": 30}
    for column, width in widths.items():
        ws.column_dimensions[column].width = width
    for row in ws.iter_rows(min_row=2):
        for cell in row:
            cell.alignment = Alignment(vertical="top", wrap_text=True)

    guide = workbook.create_sheet("guidelines")
    guidance = [
        ("목적", "ROUGE recall 65 평가를 위한 사람 참조 요약 2종 구축"),
        ("독립 작성", "검수자 1·2는 서로의 요약을 보지 않고 작성한다."),
        ("길이", "원문 문장 수의 약 25~35%. 필요하면 의미 유지 범위에서 조정한다."),
        ("필수", "핵심 인물, 사건, 시간·장소, 원인과 결과를 보존한다."),
        ("금지", "원문에 없는 사실·감정·평가를 추가하지 않는다."),
        ("문체", "자연스런 한국어 줄글로 작성하고 목록·제목은 쓰지 않는다."),
        ("완료", "두 요약과 검수자명을 입력한 후 status를 completed로 변경한다."),
    ]
    for key, value in guidance:
        guide.append([key, value])
    guide.column_dimensions["A"].width = 18
    guide.column_dimensions["B"].width = 100
    for cell in guide[1]:
        cell.font = Font(bold=True)

    args.out.parent.mkdir(parents=True, exist_ok=True)
    workbook.save(args.out)
    print(f"저장: {args.out} / {len(selected)}건 / 작성자 그룹 {len({r['source_group'] for r in selected})}개")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())