"""Build a frozen 1,000-source silver-reference trial; persist every model output. This is a newly defined experiment, not a reproduction of the undocumented 2025 tokenizer. Report character and whitespace bigram F1 without choosing by score. """ from __future__ import annotations import argparse import concurrent.futures import hashlib import json import random import re from collections import Counter from pathlib import Path import sys sys.path.insert(0, str(Path(__file__).resolve().parents[1])) PROMPTS = { "reference": "다음 전기 문단의 중심 사건 또는 주제를 한국어 한 문장, 50자 이내로 요약하세요. 핵심 인물과 행동 또는 원인·결과를 보존하고 원문의 표현을 가능한 한 유지하세요. 배경 설명과 수식은 줄이고 원문에 없는 사실은 쓰지 마세요. 제목·목록 없이 요약만 출력하세요.", "baseline": "다음 전기 문단을 최대 50자 이내로 요약하세요. 핵심 주제를 포함하고 불필요한 수식어를 제거하며 원문에 없는 내용을 추가하지 마세요. 요약문만 출력하세요.", "candidate": "전기 문단을 읽고 가장 중요한 인물, 핵심 사건, 그 결과를 찾아 한국어 한 문장 50자 이내로 요약하세요. 원문의 핵심 명사와 동사를 유지하고 중복·주변 묘사는 제외하세요. 사실을 추가하지 말고 핵심 사건을 구체적으로 표현하세요. 요약만 출력하세요.", } def digest(text): return hashlib.sha256(text.encode()).hexdigest() def metrics(reference, hypothesis, mode): def grams(text): tokens = list(re.sub(r"\s+", "", text)) if mode == "character" else text.split() return Counter(tuple(tokens[i:i + 2]) for i in range(len(tokens) - 1)) a, b = grams(reference), grams(hypothesis) overlap = sum((a & b).values()) recall = overlap / sum(a.values()) if a else 0.0 precision = overlap / sum(b.values()) if b else 0.0 return {"precision": precision, "recall": recall, "f1": 2 * precision * recall / (precision + recall) if precision + recall else 0.0} def main(): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--source", type=Path, required=True) ap.add_argument("--out", type=Path, required=True) ap.add_argument("--workers", type=int, default=12) args = ap.parse_args() args.out.mkdir(parents=True, exist_ok=True) manifest_path = args.out / "manifest.json" dataset_path = args.out / "sources.jsonl" if not manifest_path.exists(): groups, seen = [], set() rng = random.Random(20260916) for p in sorted(args.source.glob("*.json")): group = [] for row in json.loads(p.read_text()).get("results", []): text = str(row.get("source_text", "")).strip() h = digest(" ".join(text.split())) if not 200 <= len(text) <= 2000 or h in seen: continue seen.add(h) group.append({"source_file": p.name, "source_index": row.get("index"), "text": text, "source_sha256": h}) rng.shuffle(group) if group: groups.append(group) selected = [] while len(selected) < 1000 and any(groups): for group in groups: if group and len(selected) < 1000: selected.append(dict(group.pop(), id=len(selected) + 1)) if len(selected) != 1000: raise ValueError("Need 1,000 unique source passages") payload = "".join(json.dumps(row, ensure_ascii=False) + "\n" for row in selected) dataset_path.write_text(payload) manifest = {"count": 1000, "seed": 20260916, "source_books": len(groups), "dataset_sha256": digest(payload), "reference_origin": "AI-generated; not human-reviewed", "models": {"reference": "gpt-4o", "baseline": "gpt-4o-mini", "candidate": "gpt-4o"}, "prompts": PROMPTS, "temperature": 0.3, "max_tokens": 100, "tokenizers": {"character": "remove whitespace, preserve punctuation, Unicode characters", "word": "Python str.split, preserve punctuation"}, "n": 2, "aggregation": "macro mean F1 across all 1000 sources", "legacy_equivalence": "unconfirmed: original ngram_tokenize unavailable", "training_exclusion": "not locally fine-tuned; foundation-model training overlap unknown"} manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2)) manifest = json.loads(manifest_path.read_text()) payload = dataset_path.read_text() if digest(payload) != manifest["dataset_sha256"]: raise ValueError("Frozen dataset hash mismatch") rows = [json.loads(line) for line in payload.splitlines()] from app.core.config import get_settings from openai import OpenAI client = OpenAI(api_key=get_settings().openai_api_key, timeout=90, max_retries=3) outputs = {} for role in ("reference", "baseline", "candidate"): path = args.out / (role + ".jsonl") existing = [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] done = {row["id"]: row for row in existing if row.get("summary")} def generate(row): try: response = client.chat.completions.create( model=manifest["models"][role], temperature=manifest["temperature"], max_tokens=manifest["max_tokens"], messages=[ {"role": "system", "content": "당신은 텍스트 요약 전문가입니다."}, {"role": "user", "content": manifest["prompts"][role] + "\n\n" + row["text"]}]) choice = response.choices[0] summary = (choice.message.content or "").strip() return {"id": row["id"], "summary": summary, "model": response.model, "response_id": response.id, "finish_reason": choice.finish_reason, "usage": response.usage.model_dump() if response.usage else None, "over_50_chars": len(summary) > 50} except Exception as exc: return {"id": row["id"], "summary": "", "error_type": type(exc).__name__} with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool, path.open("a") as out: futures = [pool.submit(generate, row) for row in rows if row["id"] not in done] for future in concurrent.futures.as_completed(futures): result = future.result() out.write(json.dumps(result, ensure_ascii=False) + "\n") out.flush() done[result["id"]] = result if len(done) % 50 == 0: print(role, len(done), "/ 1000", flush=True) outputs[role] = done result = {"sample_count": 1000, "reference_origin": manifest["reference_origin"], "legacy_equivalence": manifest["legacy_equivalence"], "certificate_target_achieved": None, "dataset_sha256": manifest["dataset_sha256"], "scores": {}, "quality": {}} for role, items in outputs.items(): result["quality"][role] = {"empty": sum(not r.get("summary") for r in items.values()), "over_50_chars": sum(r.get("over_50_chars", False) for r in items.values()), "truncated": sum(r.get("finish_reason") == "length" for r in items.values())} per_item = [] for role in ("baseline", "candidate"): result["scores"][role] = {} for mode in ("character", "word"): scores = [] for row in rows: rid = row["id"] score = metrics(outputs["reference"][rid]["summary"], outputs[role][rid]["summary"], mode) scores.append(score) per_item.append({"id": rid, "role": role, "mode": mode, **score}) result["scores"][role][mode] = {key: sum(s[key] for s in scores) / 1000 for key in ("precision", "recall", "f1")} (args.out / "per_item.jsonl").write_text("".join(json.dumps(r) + "\n" for r in per_item)) (args.out / "result.json").write_text(json.dumps(result, ensure_ascii=False, indent=2)) print(json.dumps(result, ensure_ascii=False, indent=2), flush=True) if __name__ == "__main__": main()