{
  "status": "completed",
  "evaluation": "separate_testset_precision_not_operational_7786",
  "command": "python3 scripts/run_precision_eval.py --testset data/eval/case_precision_20260918",
  "source_testset": "data/eval/testset_v2",
  "base_commit": "52a0fdcdf041f0b7a0ef6132558b6c464b03090b",
  "worktree_changes": "case matching response fields; existing unrelated summary work preserved",
  "started_engine_version": "o2o-plagiarism-2.0.0-pdf-v1.2",
  "python": "3.14.1",
  "packages": {
    "pydantic": "2.12.5",
    "scikit-learn": "1.9.0",
    "numpy": "2.4.0",
    "kiwipiepy": "0.23.2"
  },
  "input_sha256": {
    "index.jsonl": "7512080bcaa67f373d6294cf3ab735e698261b1ba5d06ba17a6a20f5d27036a7",
    "pairs.jsonl": "aff3b80738f6a1ce72512172c0da8ca9736e91330671e252f8067baf00bceff0",
    "manifest.json": "d056756ba0cb0fcf1af13bf9634dda3ed75e45acb48b9ff69437d1b0d5a2b47d"
  },
  "manifest": {
    "seed": 20260908,
    "index_author_ratio": 0.7,
    "authors": {
      "total": 290,
      "index": 203,
      "query": 87
    },
    "counts": {
      "plagiarism": 500,
      "legitimate": 500,
      "index_segments": 4033
    },
    "plagiarism_mix": {
      "verbatim": 100,
      "partial": 100,
      "sentence_shuffle": 100,
      "lexical_swap": 150,
      "compress": 50
    },
    "hard_negatives": 150,
    "lexical_swap_rate": 0.35,
    "excluded": "대필(인칭 전환)은 계약·동의로 표절 여부가 갈려 텍스트만으로 판정할 수 없어 제외"
  },
  "operational_rerun": {
    "status": "stopped_cohort_mismatch",
    "expected": 7786,
    "observed_queries": 6343,
    "autobiography_episodes": 6331,
    "life_writing_documents": 12,
    "indexed_segments": 8025,
    "precision": null,
    "reason": "Original cohort and keyed human labels required; partial run excluded."
  },
  "result": {
    "precision": 0.9840319361277445,
    "recall": 0.986,
    "f1": 0.9850149850149851,
    "tp": 493,
    "fp": 8,
    "tn": 492,
    "fn": 7,
    "by_transformation": {
      "verbatim": {
        "tp": 100,
        "fp": 0,
        "tn": 0,
        "fn": 0
      },
      "sentence_shuffle": {
        "tp": 100,
        "fp": 0,
        "tn": 0,
        "fn": 0
      },
      "legitimate": {
        "tp": 0,
        "fp": 1,
        "tn": 349,
        "fn": 0
      },
      "lexical_swap": {
        "tp": 144,
        "fp": 0,
        "tn": 0,
        "fn": 6
      },
      "partial": {
        "tp": 99,
        "fp": 0,
        "tn": 0,
        "fn": 1
      },
      "compress": {
        "tp": 50,
        "fp": 0,
        "tn": 0,
        "fn": 0
      },
      "hard_negative": {
        "tp": 0,
        "fp": 7,
        "tn": 143,
        "fn": 0
      }
    },
    "criteria": {
      "similarity": 0.65,
      "min_exact_span": 35,
      "min_coverage": 0.3,
      "require_exact_span_evidence": true
    }
  },
  "tests": {
    "passed": 57,
    "command": "python3 -m pytest tests/test_case_matching_views.py tests/test_pdf_compliance.py tests/test_case_coverage.py tests/test_api.py tests/test_legal_risk.py tests/test_review_regressions.py -q"
  },
  "scorecard": "reports/CASE_MATCHING_PRECISION_SCORECARD_20260918.json"
}