52 lines
1.9 KiB
Python
52 lines
1.9 KiB
Python
#!/usr/bin/env python3
|
|
"""PDF/DOCX 원본을 페이지/문단 provenance와 함께 SQLite에 적재한다."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
if __package__ in (None, ""):
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from app.engine.provenance import CorpusStore
|
|
from app.engine.source_extraction import extract_source
|
|
|
|
|
|
def main() -> int:
|
|
p = argparse.ArgumentParser(description=__doc__)
|
|
p.add_argument("source_dir", type=Path)
|
|
p.add_argument("--database", type=Path, required=True)
|
|
p.add_argument("--chunk-size", type=int, default=1000)
|
|
p.add_argument("--stride", type=int, default=500)
|
|
args = p.parse_args()
|
|
if not args.source_dir.is_dir():
|
|
p.error(f"source directory does not exist: {args.source_dir}")
|
|
store = CorpusStore(args.database)
|
|
files = sorted(
|
|
path for path in args.source_dir.rglob("*")
|
|
if path.is_file() and path.suffix.lower() in {".pdf", ".docx"}
|
|
)
|
|
report = {"files": len(files), "documents": 0, "inserted": 0, "duplicates": 0,
|
|
"warnings": [], "failures": []}
|
|
for path in files:
|
|
try:
|
|
result = extract_source(path, args.chunk_size, args.stride)
|
|
store.upsert_document(result.document)
|
|
inserted, duplicates = store.add_segments(result.segments)
|
|
report["documents"] += 1
|
|
report["inserted"] += inserted
|
|
report["duplicates"] += duplicates
|
|
report["warnings"].extend(f"{path.name}: {w}" for w in result.warnings)
|
|
except Exception as exc:
|
|
report["failures"].append({"file": str(path), "error": str(exc)})
|
|
report["store"] = store.stats()
|
|
print(json.dumps(report, ensure_ascii=False, indent=2))
|
|
return 1 if report["failures"] else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|