diff --git a/reports/dedup_exclude_segments.txt b/reports/dedup_exclude_segments.txt new file mode 100644 index 0000000..a64cb64 --- /dev/null +++ b/reports/dedup_exclude_segments.txt @@ -0,0 +1,256 @@ +seg-d3fbe94228ab881296b6 +seg-751c3589b594278c083a +seg-5fc0614ce99a6618c8df +seg-e9d6cfe7d8c32ac76685 +seg-5661be946f8b631f15a5 +seg-075a0e34ae4297d7a690 +seg-50db79ea6d2727c2294b +seg-247b665d9d4ee02b33f9 +seg-44984b4a4e98fa4a523c +seg-07581bfbc6837377918b +seg-bff759c20055d96d1f4b +seg-f5c5bca93e58b039a022 +seg-f5d329cc2b30ae71f105 +seg-4d6b39d38d78d612702c +seg-0edf0ca49a261c64c385 +seg-55dde76d300a6ae8c3de +seg-ffdde9a61951d9ebb1a3 +seg-410f0ebe300e69c627b9 +seg-1245e54f50bc293ccaf6 +seg-8331dded31e15a735156 +seg-6f4f844cfa81f266abce +seg-8c6cacccd47732ef2df3 +seg-0f3197e20379dd548cc3 +seg-9a98e4bbf3c132bb9b5b +seg-23ded3d3143f049f3082 +seg-e37c0908ca2aa3f55503 +seg-24f8107434fc48c067b1 +seg-5d4f783c3645437c6070 +seg-5274f32e15cf95d178d0 +seg-d243f0147bf6b556f572 +seg-a467d7d4e7c1fb53da80 +seg-cc9363262d7bb9159306 +seg-5d7f16828df7db4a2b76 +seg-54022db9a1d633a4045d +seg-d0b69f55d3f934f87b5f +seg-bdb4188636fcf0d96ca4 +seg-288dc6c99554be21fa18 +seg-52d8fcc2f28a5ee0d224 +seg-83d63d8d706faefd7eb5 +seg-cf0cdf656e0fb06ef873 +seg-eb77d16bc244ad727b84 +seg-30d7e35a031a46cc71e0 +seg-a42a048a63356af43586 +seg-5ff84bb1c602e8410ed7 +seg-bbd6204bfd92ce23ae88 +seg-6a24f151bbc172428608 +seg-b71d5e09f89dd7c86899 +seg-d1830d7e5b83dce69c15 +seg-bf88dc382fa57030f641 +seg-be8c1f14df73996ff3fc +seg-22a696af220d87a59378 +seg-ac6d6a1c976b9d03b8a1 +seg-d383082fce6037b85412 +seg-9dbdda1626f8269a4956 +seg-ee361f0c77f7f91b9d5d +seg-3ef361f6e3f6d8238949 +seg-dde5178ca77977d2af77 +seg-430ce38f4aeaec2ee30d +seg-dc3ed111c198333b41a6 +seg-62dbfefbb87ae7edffb5 +seg-70c32fa5c3fa064d8517 +seg-38ce69b80bff6bf91bad +seg-aa5a97c20e56d3e06a18 +seg-a538c6ed532265042c2c +seg-8b80b03f94f0ff0a09fc +seg-ef8b9a90adaabae27467 +seg-b88a0f96c237d9047883 +seg-e26a0aec6bfa1ac48242 +seg-9371d04bedc46ff3307a +seg-23bd6e7863e72a037696 +seg-4527a642f6740fb1990d +seg-2e774b13ce5649bbef44 +seg-7a544a5709a515eebe9f +seg-f5b34bfd2d2808b30d57 +seg-a5827f00cc888d6bd553 +seg-e3e107d84c4705a9cce1 +seg-53fe28079bf8e65376da +seg-8c2c713c44ef651f5138 +seg-d182c9b4a909dd0a7bbb +seg-93ceac7aa9a9e5a06ce1 +seg-a398d393288e13d66de3 +seg-52cb8da0dc336319f732 +seg-ae11015b4ae9b766d5c3 +seg-30b280d1c2859a5ae036 +seg-9c13ba45ae839ebfc44d +seg-5600dc312d9effe82322 +seg-cd270243863ef7841dfb +seg-3f5f830108e2bdaeefd4 +seg-516a947f58a9a8b168d8 +seg-cf8f63bf5fe850471401 +seg-7507defdcb9416205954 +seg-f7e0090d229fc1a308b0 +seg-e0d851a207476f87c789 +seg-bd5662c91ab35270516b +seg-be04976635524964ce62 +seg-9a33bce7c80b6fb1248e +seg-c9fdd79c5936c73355c7 +seg-fa44f8a2635163b24fb7 +seg-62d41db76b35c50fc80c +seg-943fd1f2254a41d56795 +seg-9dcd69c2d11e6cca179e +seg-7d35c0b29a378389e352 +seg-76b19ec66f94dffaf00a +seg-e66251e5a312bc798e2e +seg-a9b38449e85ed466047e +seg-abf6d9be2a35224b9e09 +seg-aab80bc4d6a9f419c723 +seg-bd7b1fd2bc26447c9910 +seg-8da64f864be5944aedef +seg-587d9bf5b0eca24fe707 +seg-9eb5bd09ab63980559b4 +seg-ee19e6aaf1b089d3e641 +seg-8a7b17ec37d128136ac9 +seg-4504aa1d788d212d2df5 +seg-b98614064b81140e0fe4 +seg-966a6311fb356d92b6d1 +seg-bc2e4bb72583ecf07179 +seg-576943b2975470189554 +seg-f773aa3058f06ae79b47 +seg-f4be6c5c541c902d642b +seg-a3fca07ea170e36417e2 +seg-c5352306e93499c0a97b +seg-7c843b92edcd5b2a9fd9 +seg-81c3bd5d41f4a068de6e +seg-8f78bae75bb369f94503 +seg-e0de5524dfe1159eb363 +seg-ef54adc86f4a0a297118 +seg-bedd4f81482541351bb7 +seg-e811cffbcc76695b6e6f +seg-c3b58a34306a6a4e5b59 +seg-bf75b74deb39ccf1303a +seg-c8d5a76ff7e035ef22b6 +seg-b8637db3c35177137a3f +seg-ccaf5c07904036512566 +seg-7419176fd92cceea9686 +seg-b72eea0366a31e987386 +seg-69ea27fd2881c7627ded +seg-ad4a1cc4ff85607b6c66 +seg-f7a461e0a15fb18ba98f +seg-896990d33e817107c0bc +seg-8cecf2cdee3da79b1b05 +seg-f885a8e028dadc75d8e2 +seg-e83d73a7381c3c9d52e1 +seg-725a57d77c855fe2c405 +seg-d9b963dc38c6336c267f +seg-a8facc3763ded9d2ddb1 +seg-fafe437f7069e5fd7722 +seg-7bf545545753af04309f +seg-af0a392936f76d3f0bbc +seg-5956baedf58f802290ca +seg-8f64481ef86947095f9c +seg-68644acb7b741ed0040f +seg-f5a3280bab5b83a6716b +seg-e00351c4a290824c305c +seg-881fe2194708c84ff2bc +seg-905c5684472d150fbaf8 +seg-9977b26f515658764743 +seg-f25987b145b277afd1bb +seg-8620bf9b61e70258c169 +seg-b94936f18a528d1d1484 +seg-78b533ef1ae0d6edae2d +seg-f4dd4bb7814c681310be +seg-ef082f2347172e8cecf4 +seg-8ad3bcfb1e19146479aa +seg-d9651090b7a6aa2f0430 +seg-e1e78ffdc76fd9ffaf9e +seg-cff3db09ecfb8d7fc0d7 +seg-a6355ffad5bd71af53c6 +seg-d53dda8d81b2adf120e8 +seg-67563b9afa606c5587c7 +seg-c43016a4caaf3143cd98 +seg-a0837f25f895120a065f +seg-c9da95e71f1e2fabd0ec +seg-fbb309c63c26989c8576 +seg-c29909ee9a58cb26b01a +seg-819aecc9f54fd9b55be2 +seg-d0a0f26c8be62bfb2d0c +seg-fd26051079a4bf040532 +seg-98a5697519e5f97e249a +seg-d2751017aa736359f7d4 +seg-74fed19b12664e50be9c +seg-c23c2fc3639889509e9e +seg-c36dff9dde138672ddee +seg-e3c87966f3b3f2bb6841 +seg-e4771ac5792eeb914b07 +seg-cd775fabaf6cb4994cdf +seg-7a1c0983f84940c7201b +seg-9d79705b707d9e32b117 +seg-b0c16c55fa9e8277349d +seg-d57e517755b00919daca +seg-79b6ba59c8781f00ce0a +seg-f6c8922e9d20de2076ff +seg-c93928d4782eb4c6afe8 +seg-93bcd6a33398bd7280ec +seg-f832f4d3a11932fd556a +seg-d6810579afc033bd1b14 +seg-b6ed3f352a7847b0b8a1 +seg-a7358497f1e85ad51ff8 +seg-e17e714288dc109c5baa +seg-a1655a23565dc471aa47 +seg-d5fe534027eb9c1e85fd +seg-ab243504653f9648e7bf +seg-dfec7d5d1b119a418561 +seg-bf99840a40b62052271b +seg-a985637ec6ab3d46d555 +seg-a9413062361aa4f345c4 +seg-e57636f9636d23ef4039 +seg-9554999648a9aadc31bd +seg-c62f06ad5656a4599475 +seg-b49e6da1a3a6b0eba049 +seg-d7063bf4d41293c25c90 +seg-ccf95d993d1c472e27e9 +seg-b7f22ef0b4697779bde5 +seg-bbce92de4976f5dc93a1 +seg-f03532de0097e52edb9f +seg-f70fdab6a38ff5140308 +seg-c23074b3203f30fbba5c +seg-d6ea9afd3697d2aaefe1 +seg-d07c0d0cd0d0aea3cfc9 +seg-eead2327f235ebd6efe7 +seg-9fd1b8cd453902242690 +seg-a82cd00ce71260f0350a +seg-a5a838ab500c1da7e11a +seg-a311b6e8f8c03dc79c1e +seg-fff1369f0ea4f9f17516 +seg-d3dbae3326a84db54529 +seg-ed34c62c55263092b07e +seg-d35338514edde20b18b5 +seg-aa007367cea2c002f094 +seg-d0b236f8748a269bdecc +seg-ed816d3c437985f03ea5 +seg-debed50cad3a7946a2cf +seg-eaacca81f4591e799863 +seg-fda2d67cf5a23cb91b3e +seg-e7ad9915d5e81d735cb0 +seg-df4cfebb17d4574beaf6 +seg-c0e7b6315ea41307ea0d +seg-cb9c60dd9ccad9606a72 +seg-fb900d525ce3b5fac463 +seg-e75049a453021a03ff30 +seg-e01166955fe0dadec87d +seg-fccf864c1bd61629218c +seg-d8ecd0403ec0c7ddbdbb +seg-fecff1ad96f117b29fb9 +seg-ea129bba581d8ab62b7a +seg-da7ac80c29ff726d58ab +seg-dd7f88929af72ef109e2 +seg-fdc6d5861c7d7e2edee9 +seg-f923d6974271cb91fb5a +seg-f1c0cf14a231160b7f3e +seg-e97c69c482cd66aa04cd +seg-f552457bdb5adcf726fe +seg-ecbbb6b9ee769091fdb5 +seg-f9da50c0714c3e8eca0a +seg-ec05c1d86aacdb87b00f +seg-f97a6835ca56b8bcf03d diff --git a/scripts/build_duplicate_report.py b/scripts/build_duplicate_report.py new file mode 100644 index 0000000..becd78d --- /dev/null +++ b/scripts/build_duplicate_report.py @@ -0,0 +1,175 @@ +"""코퍼스 안의 중복 원고를 찾아 목록과 제외 대상 세그먼트를 만든다. + +중복 제거 후 재실행하면 중복에서 비롯된 침해의심 건이 결과에서 사라진다. +사라지기 전에 그 건들이 어떤 문서 사이의 중복이었는지 남겨두어야, 나중에 +같은 제출자의 중복인지 다른 제출자의 표절인지 확인할 수 있다. + +산출물 + - XLSX : 중복 그룹 목록과, 그 중복 때문에 침해의심이 된 건의 원문 + - TXT : 재실행 시 제외할 segment_id 목록 (그룹마다 첫 건만 남긴다) +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from collections import defaultdict +from pathlib import Path + +from openpyxl import Workbook +from openpyxl.styles import Alignment, Font, PatternFill +from openpyxl.utils import get_column_letter + +HEADER_FILL = PatternFill("solid", fgColor="D9E1F2") +NOTE_FONT = Font(italic=True, color="7F7F7F") + + +def normalize(text: str) -> str: + """공백 차이만 있는 원고를 같은 것으로 본다.""" + return re.sub(r"\s+", "", text or "") + + +def text_key(text: str) -> str: + return hashlib.sha256(normalize(text).encode("utf-8")).hexdigest()[:16] + + +def load_jsonl(path: Path) -> dict[str, dict]: + rows = {} + for line in path.read_text(encoding="utf-8").splitlines(): + if line.strip(): + row = json.loads(line) + rows[row["query_key"]] = row + return rows + + +def write_header(sheet, columns) -> None: + sheet.append([label for label, _ in columns]) + for index, (_, width) in enumerate(columns, start=1): + cell = sheet.cell(row=1, column=index) + cell.font = Font(bold=True) + cell.fill = HEADER_FILL + sheet.column_dimensions[get_column_letter(index)].width = width + sheet.freeze_panes = "A2" + + +GROUP_COLUMNS = [ + ("중복 그룹", 10), ("중복 건수", 10), ("문서 수", 9), ("성격", 22), + ("가명 제목들", 40), ("문서 ID들", 60), ("본문 길이", 10), ("본문", 80), +] + +SUSPECT_COLUMNS = [ + ("가명 제목", 20), ("문서 ID", 28), ("매칭 상대 제목", 20), ("매칭 상대 문서 ID", 28), + ("결합유사도", 11), ("본문 완전 동일", 13), ("본문 길이", 10), + ("검사 대상 원문", 70), ("매칭 상대 원문", 70), +] + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--excerpts-jsonl", type=Path, required=True) + parser.add_argument("--batch-jsonl", type=Path, required=True) + parser.add_argument("--out-xlsx", type=Path, required=True) + parser.add_argument("--out-exclude", type=Path, required=True, + help="재실행 시 제외할 segment_id 목록 (한 줄에 하나)") + args = parser.parse_args() + + excerpts = load_jsonl(args.excerpts_jsonl) + batch = load_jsonl(args.batch_jsonl) + + groups: dict[str, list[str]] = defaultdict(list) + for key, row in excerpts.items(): + if normalize(row["검사 대상 원문"]): + groups[text_key(row["검사 대상 원문"])].append(key) + duplicates = {key: keys for key, keys in groups.items() if len(keys) > 1} + + # 그룹마다 첫 건만 남기고 나머지를 제외 대상으로 삼는다. + excluded = [key for keys in duplicates.values() for key in sorted(keys)[1:]] + segment_ids = [batch[key].get("query_segment_id") or batch[key]["doc_id"] for key in excluded] + args.out_exclude.parent.mkdir(parents=True, exist_ok=True) + args.out_exclude.write_text("\n".join(segment_ids) + "\n", encoding="utf-8") + + wb = Workbook() + wb.remove(wb.active) + + # 1) 중복 때문에 침해의심이 된 건 — 사라지기 전에 원문째로 남긴다. + sheet = wb.create_sheet("중복 기인 의심 건") + write_header(sheet, SUSPECT_COLUMNS) + affected = [ + key for key, row in excerpts.items() + if row["침해 여부"] == "침해 의심" and len(groups[text_key(row["검사 대상 원문"])]) > 1 + ] + for key in sorted(affected, key=lambda k: -(excerpts[k]["결합유사도"] or 0)): + row, meta = excerpts[key], batch[key] + identical = normalize(row["검사 대상 원문"]) == normalize(row["매칭 상대 원문"]) + sheet.append([ + row["가명 제목"], meta["doc_id"], row.get("매칭 상대 제목") or "-", + meta.get("매칭 상대 doc") or "-", row["결합유사도"], + "예" if identical else "아니오", len(normalize(row["검사 대상 원문"])), + row["검사 대상 원문"], row["매칭 상대 원문"], + ]) + for column in (8, 9): + sheet.cell(row=sheet.max_row, column=column).alignment = Alignment( + wrap_text=True, vertical="top") + sheet.cell(row=sheet.max_row, column=5).number_format = "0.0000" + sheet.auto_filter.ref = f"A1:{get_column_letter(len(SUSPECT_COLUMNS))}{sheet.max_row}" + + # 2) 중복 그룹 전체 목록 + sheet = wb.create_sheet("중복 그룹 전체") + write_header(sheet, GROUP_COLUMNS) + for index, (_, keys) in enumerate( + sorted(duplicates.items(), key=lambda item: (-len(item[1]), item[0])), start=1 + ): + docs = sorted({batch[key]["doc_id"] for key in keys}) + titles = sorted({excerpts[key]["가명 제목"] for key in keys}) + body = excerpts[keys[0]]["검사 대상 원문"] + sheet.append([ + index, len(keys), len(docs), + "서로 다른 문서 사이" if len(docs) > 1 else "같은 문서 안 (적재 오류)", + ", ".join(titles), ", ".join(docs), len(normalize(body)), body, + ]) + sheet.cell(row=sheet.max_row, column=8).alignment = Alignment( + wrap_text=True, vertical="top") + sheet.auto_filter.ref = f"A1:{get_column_letter(len(GROUP_COLUMNS))}{sheet.max_row}" + + # 3) 요약 + sheet = wb.create_sheet("요약") + cross = sum(1 for keys in duplicates.values() + if len({batch[key]["doc_id"] for key in keys}) > 1) + for label, value in ( + ("검사 대상 전체", len(excerpts)), + ("고유 본문", len(groups)), + ("중복 그룹", len(duplicates)), + (" 서로 다른 문서 사이", cross), + (" 같은 문서 안 (적재 오류)", len(duplicates) - cross), + ("중복에 속한 건", sum(len(keys) for keys in duplicates.values())), + ("재실행 시 제외할 건", len(excluded)), + ("제외 후 검사 대상", len(excerpts) - len(excluded)), + ("", ""), + ("침해의심 전체", sum(1 for r in excerpts.values() if r["침해 여부"] == "침해 의심")), + (" 중복에서 비롯된 건", len(affected)), + (" 중복과 무관한 건", + sum(1 for r in excerpts.values() if r["침해 여부"] == "침해 의심") - len(affected)), + ): + sheet.append([label, value]) + sheet.append([]) + for line in ( + "중복 제거는 원본 코퍼스를 지우는 것이 아니라, 재실행 시 해당 세그먼트를 검사 대상에서 빼는 것입니다.", + "'서로 다른 문서 사이'의 중복은 같은 제출자가 두 번 낸 것일 수도, 다른 제출자의 표절일 수도 있습니다.", + "후자라면 그것이 곧 침해 사례이므로, 이 목록의 문서 출처를 확인한 뒤 판단해야 합니다.", + ): + sheet.append([line]) + sheet.cell(row=sheet.max_row, column=1).font = NOTE_FONT + sheet.column_dimensions["A"].width = 30 + sheet.column_dimensions["B"].width = 14 + + args.out_xlsx.parent.mkdir(parents=True, exist_ok=True) + wb.save(args.out_xlsx) + print(f"wrote {args.out_xlsx}") + print(f"wrote {args.out_exclude} ({len(segment_ids)}건 제외 대상)") + print(f"중복 그룹 {len(duplicates)} / 중복 기인 의심 건 {len(affected)}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_embedding_infringement_batch.py b/scripts/run_embedding_infringement_batch.py index 55fc904..77ac826 100644 --- a/scripts/run_embedding_infringement_batch.py +++ b/scripts/run_embedding_infringement_batch.py @@ -40,7 +40,7 @@ def build_or_load(store, index_dir: Path, model_name: str, device: str, batch: i return vectors, meta, model def main(): - p=argparse.ArgumentParser(); p.add_argument("--database",type=Path,required=True); p.add_argument("--index-dir",type=Path,required=True); p.add_argument("--out-jsonl",type=Path,required=True); p.add_argument("--out-xlsx",type=Path,required=True); p.add_argument("--first-jsonl",type=Path,required=True); p.add_argument("--model",default="BM-K/KoSimCSE-roberta-multitask"); p.add_argument("--batch-size",type=int,default=64); p.add_argument("--progress-every",type=int,default=25); a=p.parse_args() + p=argparse.ArgumentParser(); p.add_argument("--database",type=Path,required=True); p.add_argument("--index-dir",type=Path,required=True); p.add_argument("--out-jsonl",type=Path,required=True); p.add_argument("--out-xlsx",type=Path,required=True); p.add_argument("--first-jsonl",type=Path,required=True); p.add_argument("--model",default="BM-K/KoSimCSE-roberta-multitask"); p.add_argument("--batch-size",type=int,default=64); p.add_argument("--progress-every",type=int,default=25); p.add_argument("--exclude-segments",type=Path,help="검사·색인에서 뺄 segment_id 목록 파일 (중복 제거용)"); p.add_argument("--min-exact-span",type=int,help="연속 일치 판정 기준(공백 포함 글자 수). 미지정 시 설정값"); a=p.parse_args() logging.basicConfig(level=logging.INFO,format="%(asctime)s %(levelname)s %(message)s") if os.getenv("USE_LLM_LEGAL_JUDGE", "").lower() not in ("", "0", "false", "no", "off"): raise RuntimeError("USE_LLM_LEGAL_JUDGE=false required") import torch @@ -49,7 +49,20 @@ def main(): store=CorpusStore(a.database); vectors,meta,model=build_or_load(store,a.index_dir,a.model,device,a.batch_size) get_settings.cache_clear(); settings=get_settings().model_copy(update={"corpus_db_path":str(a.database),"persistent_index_dir":"/nonexistent","use_persistent_index":False,"use_llm_legal_judge":False}) # 법령/태그/규칙 엔진만 재사용한다. CPU 운영 인덱스를 변경하지 않는다. + if a.min_exact_span: + settings=settings.model_copy(update={"persistent_min_exact_span":a.min_exact_span}) + LOG.info("min_exact_span=%d (기본값 대신 적용)",a.min_exact_span) detector=PlagiarismDetector(settings); groups=store.document_source_groups(); records=store.get_segments(meta["segment_ids"]); items=build_query_plan(store) + # 중복 제거: 검사 대상과 색인 후보에서 동시에 뺀다. 한쪽만 빼면 지운 원고가 + # 여전히 상대로 잡혀 중복이 결과에 남는다. + if a.exclude_segments: + drop={x.strip() for x in a.exclude_segments.read_text(encoding="utf-8").splitlines() if x.strip()} + before=len(items); items=[it for it in items if (it.source_segment_id or it.document_id) not in drop] + indexed=len(meta["segment_ids"]); keep_ix=[i for i,sid in enumerate(meta["segment_ids"]) if sid not in drop] + vectors=vectors[keep_ix] + meta={**meta,"segment_ids":[meta["segment_ids"][i] for i in keep_ix], + "segment_document_ids":[meta["segment_document_ids"][i] for i in keep_ix]} + LOG.info("dedup: 질의 %d->%d, 색인 %d->%d",before,len(items),indexed,len(keep_ix)) a.out_jsonl.parent.mkdir(parents=True,exist_ok=True); started=time.monotonic(); rows=[] for n,item in enumerate(items,1): qv=np.asarray(model.encode([item.text[:2048]],normalize_embeddings=True,show_progress_bar=False,convert_to_numpy=True)[0],dtype=np.float32); scores=vectors@qv