124 lines
4.7 KiB
Python
124 lines
4.7 KiB
Python
#!/usr/bin/env python3
|
|
"""로컬 생활수기 TXT를 익명화해 런타임 코퍼스에 적재한다.
|
|
|
|
원본 파일명·작성자 이름은 SQLite에 넣지 않는다. 파일별 문서/작성자 그룹은
|
|
운영 salt 기반 가명으로만 보존하고, 본문의 이메일·전화·주민번호 형식은 제거한다.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
if __package__ in (None, ""):
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from app.engine.provenance import CorpusStore, DocumentRecord, SegmentRecord, stable_id
|
|
from app.engine.source_extraction import chunk_with_offsets
|
|
from app.engine.training_data import pseudonymous_id, redact_direct_identifiers
|
|
|
|
|
|
def file_sha256(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as stream:
|
|
for block in iter(lambda: stream.read(1024 * 1024), b""):
|
|
digest.update(block)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("source_dir", type=Path)
|
|
parser.add_argument("--database", type=Path, required=True)
|
|
parser.add_argument("--anonymize", action="store_true")
|
|
parser.add_argument("--anonymization-salt-env", default="DATA_ANONYMIZATION_SALT")
|
|
parser.add_argument("--provenance", default="life_writing")
|
|
parser.add_argument("--chunk-size", type=int, default=1000)
|
|
parser.add_argument("--stride", type=int, default=500)
|
|
args = parser.parse_args()
|
|
if not args.source_dir.is_dir():
|
|
parser.error(f"source directory does not exist: {args.source_dir}")
|
|
if not args.anonymize:
|
|
parser.error("생활수기 적재에는 --anonymize 가 필요합니다")
|
|
salt = os.environ.get(args.anonymization_salt_env, "")
|
|
if not salt:
|
|
parser.error(f"{args.anonymization_salt_env} 환경변수가 필요합니다")
|
|
|
|
files = sorted(path for path in args.source_dir.glob("*.txt") if path.is_file())
|
|
if not files:
|
|
parser.error(".txt 원문이 없습니다")
|
|
store = CorpusStore(args.database)
|
|
documents: list[DocumentRecord] = []
|
|
segments: list[SegmentRecord] = []
|
|
skipped_empty = 0
|
|
redaction_changes = 0
|
|
|
|
for path in files:
|
|
raw = path.read_text(encoding="utf-8", errors="replace")
|
|
clean = redact_direct_identifiers(raw).strip()
|
|
if not clean:
|
|
skipped_empty += 1
|
|
continue
|
|
raw_hash = file_sha256(path)
|
|
document_id = pseudonymous_id(f"life-writing:{raw_hash}", salt, prefix="doc")
|
|
source_group = pseudonymous_id(f"life-writing:{raw_hash}", salt, prefix="source")
|
|
display_title = f"익명 생활수기 {document_id.split(':', 1)[-1][:8]}"
|
|
documents.append(DocumentRecord(
|
|
document_id=document_id,
|
|
title=display_title,
|
|
metadata={
|
|
"provenance": args.provenance,
|
|
"corpus_kind": "life_writing",
|
|
"source_group": source_group,
|
|
"human_verified": True,
|
|
"ai_assistance": False,
|
|
"coordinate_scope": "document",
|
|
"anonymized": True,
|
|
},
|
|
))
|
|
if clean != raw.strip():
|
|
redaction_changes += 1
|
|
for ordinal, (start, end, chunk) in enumerate(
|
|
chunk_with_offsets(clean, size=args.chunk_size, stride=args.stride), 1
|
|
):
|
|
segments.append(SegmentRecord(
|
|
segment_id=stable_id("seg", document_id, str(start), chunk),
|
|
document_id=document_id,
|
|
text=chunk,
|
|
ordinal=str(ordinal),
|
|
coordinate_scope="document",
|
|
char_start=start,
|
|
char_end=end,
|
|
source_locator=f"life_writing://{document_id}#chars={start}-{end}",
|
|
metadata={
|
|
"provenance": args.provenance,
|
|
"corpus_kind": "life_writing",
|
|
"human_verified": True,
|
|
"ai_assistance": False,
|
|
"provenance_quality": "text_extraction_document_offsets",
|
|
},
|
|
))
|
|
|
|
store.upsert_documents(documents)
|
|
inserted, duplicates = store.add_segments(segments)
|
|
print(json.dumps({
|
|
"files": len(files),
|
|
"documents": len(documents),
|
|
"inserted_segments": inserted,
|
|
"duplicate_segments": duplicates,
|
|
"skipped_empty_files": skipped_empty,
|
|
"files_with_direct_identifier_redaction": redaction_changes,
|
|
"anonymized": True,
|
|
"provenance": args.provenance,
|
|
"store": store.stats(),
|
|
}, ensure_ascii=False, indent=2))
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|