#!/usr/bin/env python3 """SQLite 코퍼스의 CPU 영속 후보 인덱스를 신규 구축하거나 증분 동기화한다.""" from __future__ import annotations import argparse import json import sys from pathlib import Path if __package__ in (None, ""): sys.path.insert(0, str(Path(__file__).resolve().parents[1])) from app.engine.persistent_index import PersistentCorpusIndex from app.engine.provenance import CorpusStore def precompute_features(store: CorpusStore, batch: int = 500) -> int: """참조 lemma/요소를 미리 계산해 DB 에 저장 (#5). 이 작업을 인덱싱 때 1회 해두면, 탐지 요청마다 후보 세그먼트를 형태소 분석하던 비용이 사라진다. 이미 채워진 세그먼트는 건너뛴다. """ from app.engine.extractor import get_extractor from app.engine.structural import extract_lemmas extractor = get_extractor() pending: list[tuple[str, list[str], dict]] = [] updated = 0 for segment in store.iter_segments(): if segment.lemmas is not None and segment.elements is not None: continue pending.append(( segment.segment_id, extract_lemmas(segment.text), extractor.extract(segment.text).model_dump(), )) if len(pending) >= batch: updated += store.update_segment_features(pending) print(f" 특징 계산 {updated}건…", flush=True) pending = [] if pending: updated += store.update_segment_features(pending) return updated def main() -> int: p = argparse.ArgumentParser(description=__doc__) p.add_argument("--database", type=Path, required=True) p.add_argument("--index-dir", type=Path, required=True) p.add_argument("--features", type=int, default=2**20) p.add_argument( "--skip-precompute", action="store_true", help="참조 lemma/요소 사전계산을 건너뛴다(질의 시 계산 후 백필됨)", ) args = p.parse_args() if not args.database.exists(): p.error(f"database does not exist: {args.database}") store = CorpusStore(args.database) precomputed = 0 if not args.skip_precompute: precomputed = precompute_features(store) result = PersistentCorpusIndex(args.database, args.index_dir).sync(args.features) result["precomputed_features"] = precomputed result["missing_features"] = store.count_missing_features() print(json.dumps(result, ensure_ascii=False, indent=2)) return 0 if __name__ == "__main__": raise SystemExit(main())