165 lines
6.5 KiB
Python
165 lines
6.5 KiB
Python
#!/usr/bin/env python3
|
|
"""수령한 O2O XLSX를 provenance SQLite 코퍼스로 적재한다."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import sys
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
if __package__ in (None, ""):
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from app.engine.provenance import CorpusStore, DocumentRecord, SegmentRecord, stable_id
|
|
from app.engine.training_data import (
|
|
pseudonymous_id,
|
|
redact_direct_identifiers,
|
|
sanitize_prompt_metadata,
|
|
)
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("xlsx", type=Path)
|
|
p.add_argument("--database", type=Path, required=True)
|
|
p.add_argument("--sheet", default=None, help="기본값: 첫 번째 시트")
|
|
p.add_argument("--book-column", default="book_name")
|
|
p.add_argument("--text-column", default="에피소드")
|
|
p.add_argument("--index-column", default="episode_index")
|
|
p.add_argument("--author-column", default=None,
|
|
help="작성자 그룹 컬럼. 지정하면 같은 작성자의 여러 자서전을 함께 묶는다")
|
|
p.add_argument("--episode-title-column", default=None)
|
|
p.add_argument("--path-column", default="json_path")
|
|
p.add_argument("--anonymize", action="store_true",
|
|
help="작성자·책 제목을 가명화하고 본문의 직접 식별자를 제거")
|
|
p.add_argument("--anonymization-salt-env", default="DATA_ANONYMIZATION_SALT")
|
|
p.add_argument("--provenance", default="combooks_confirmed_human")
|
|
return p.parse_args()
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
try:
|
|
import openpyxl
|
|
except ImportError:
|
|
print("openpyxl이 필요합니다: pip install openpyxl", file=sys.stderr)
|
|
return 2
|
|
if not args.xlsx.is_file():
|
|
print(f"파일을 찾을 수 없습니다: {args.xlsx}", file=sys.stderr)
|
|
return 2
|
|
|
|
wb = openpyxl.load_workbook(args.xlsx, read_only=True, data_only=True)
|
|
ws = wb[args.sheet] if args.sheet else wb.worksheets[0]
|
|
rows = ws.iter_rows(values_only=True)
|
|
headers = [str(v).strip() if v is not None else "" for v in next(rows)]
|
|
positions = {name: i for i, name in enumerate(headers)}
|
|
required = [args.book_column, args.text_column]
|
|
if args.author_column:
|
|
required.append(args.author_column)
|
|
missing = [name for name in required if name not in positions]
|
|
if missing:
|
|
print(f"필수 열 없음: {missing}; 실제 열={headers}", file=sys.stderr)
|
|
return 2
|
|
|
|
store = CorpusStore(args.database)
|
|
store.initialize()
|
|
skipped = 0
|
|
book_counts: Counter[str] = Counter()
|
|
documents: dict[str, DocumentRecord] = {}
|
|
segments: list[SegmentRecord] = []
|
|
salt = os.environ.get(args.anonymization_salt_env, "")
|
|
if args.anonymize and not salt:
|
|
print(
|
|
f"--anonymize 사용 시 {args.anonymization_salt_env} 환경변수가 필요합니다.",
|
|
file=sys.stderr,
|
|
)
|
|
return 2
|
|
|
|
for rownum, row in enumerate(rows, start=2):
|
|
book = str(row[positions[args.book_column]] or "").strip()
|
|
text = str(row[positions[args.text_column]] or "").strip()
|
|
ordinal = str(row[positions[args.index_column]] or "").strip() \
|
|
if args.index_column in positions else str(rownum - 1)
|
|
if not book or not text:
|
|
skipped += 1
|
|
continue
|
|
author = ""
|
|
if args.author_column:
|
|
author = str(row[positions[args.author_column]] or "").strip()
|
|
raw_group = f"{author}\x1f{book}" if author else book
|
|
author_group = pseudonymous_id(author, salt) if args.anonymize and author else None
|
|
document_id = (
|
|
pseudonymous_id(raw_group, salt, prefix="doc")
|
|
if args.anonymize else stable_id("doc", raw_group)
|
|
)
|
|
display_title = f"익명 자서전 {document_id.split(':')[-1][:8]}" if args.anonymize else book
|
|
if args.anonymize:
|
|
text = redact_direct_identifiers(text)
|
|
source_path = None
|
|
if args.path_column in positions and not args.anonymize:
|
|
source_path = str(row[positions[args.path_column]] or "").strip() or None
|
|
documents[document_id] = DocumentRecord(
|
|
document_id=document_id,
|
|
title=display_title,
|
|
source_path=source_path,
|
|
metadata={
|
|
"import_source": args.xlsx.name,
|
|
"provenance": args.provenance,
|
|
"human_verified": True,
|
|
"ai_assistance": False,
|
|
**({"author_group": author_group} if author_group else {}),
|
|
},
|
|
)
|
|
episode_title = None
|
|
if args.episode_title_column in positions:
|
|
episode_title = str(row[positions[args.episode_title_column]] or "").strip() or None
|
|
if episode_title and args.anonymize:
|
|
episode_title = sanitize_prompt_metadata(episode_title)
|
|
segments.append(SegmentRecord(
|
|
segment_id=stable_id("seg", document_id, text),
|
|
document_id=document_id,
|
|
text=text,
|
|
ordinal=ordinal,
|
|
coordinate_scope="episode",
|
|
char_start=0,
|
|
char_end=len(text),
|
|
source_locator=f"{source_path or args.xlsx.name}#episode={ordinal}",
|
|
metadata={
|
|
"provenance_quality": "episode_only",
|
|
"page_offset_available": False,
|
|
"provenance": args.provenance,
|
|
"human_verified": True,
|
|
"ai_assistance": False,
|
|
**({"episode_title": episode_title} if episode_title else {}),
|
|
},
|
|
))
|
|
book_counts[document_id] += 1
|
|
|
|
store.upsert_documents(documents.values())
|
|
inserted, duplicates = store.add_segments(segments)
|
|
|
|
report = {
|
|
"database": str(args.database),
|
|
"sheet": ws.title,
|
|
"inserted_segments": inserted,
|
|
"duplicate_segments": duplicates,
|
|
"skipped_rows": skipped,
|
|
"source_books": len(book_counts),
|
|
"anonymized": args.anonymize,
|
|
"provenance": args.provenance,
|
|
"store": store.stats(),
|
|
"location_warning": (
|
|
"수령 XLSX에는 원본 페이지/문단 offset이 없어 episode 좌표만 저장했습니다. "
|
|
"PDF/DOCX 재추출 전에는 페이지 근거를 표시할 수 없습니다."
|
|
),
|
|
}
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|