156 lines
5.3 KiB
Python
156 lines
5.3 KiB
Python
import json
|
|
from types import SimpleNamespace
|
|
|
|
from app.engine.legal_risk import (
|
|
LegalJudgeDecision,
|
|
LegalRiskEngine,
|
|
Precedent,
|
|
load_precedents,
|
|
)
|
|
from app.engine.openai_legal_judge import OpenAILegalJudge
|
|
|
|
|
|
def _case() -> Precedent:
|
|
return Precedent(
|
|
case_id="case-1",
|
|
title="테스트 판례",
|
|
source_url="https://example.test/case-1",
|
|
work_types=("literary",),
|
|
legal_tags=("reproduction",),
|
|
criteria=("실질적 유사성",),
|
|
holding_summary="테스트 요약",
|
|
)
|
|
|
|
|
|
def test_empty_precedents_never_claim_legal_conclusion():
|
|
result = LegalRiskEngine([]).assess(
|
|
max_similarity=0.95, coverage=0.8, longest_span=300,
|
|
legal_tags=["reproduction"],
|
|
)
|
|
assert result.status == "insufficient_precedent_data"
|
|
assert result.risk_level is None
|
|
|
|
|
|
def test_risk_requires_missing_facts_and_only_registered_case_ids():
|
|
result = LegalRiskEngine([_case()]).assess(
|
|
max_similarity=0.9, coverage=0.5, longest_span=200,
|
|
legal_tags=["reproduction"],
|
|
)
|
|
assert result.status == "review_required"
|
|
assert result.risk_level == "high"
|
|
assert result.precedent_ids == ("case-1",)
|
|
assert len(result.missing_factors) == 3
|
|
|
|
|
|
def test_repository_seed_is_valid():
|
|
cases = load_precedents("data/precedents/precedents.jsonl")
|
|
assert cases
|
|
assert any(case.case_id == "2012다73493" for case in cases)
|
|
assert len(cases) == len({case.case_id for case in cases})
|
|
|
|
|
|
class _FakeJudge:
|
|
def __init__(self, case_ids=("case-1",), review_required=True):
|
|
self.case_ids = case_ids
|
|
self.review_required = review_required
|
|
self.request = None
|
|
|
|
def judge(self, request):
|
|
self.request = request
|
|
return LegalJudgeDecision(
|
|
verdict="likely",
|
|
confidence=0.82,
|
|
matched_precedent_ids=self.case_ids,
|
|
supporting_reasons=("표현 일치 구간이 길다",),
|
|
counter_reasons=("접근 가능성은 확인되지 않았다",),
|
|
missing_factors=("시장 영향",),
|
|
review_required=self.review_required,
|
|
model="test-gpt",
|
|
prompt_version="test-v1",
|
|
)
|
|
|
|
|
|
def test_llm_judge_augments_rule_result_and_keeps_deterministic_missing_factors():
|
|
judge = _FakeJudge()
|
|
result = LegalRiskEngine([_case()], judge=judge).assess(
|
|
max_similarity=0.9,
|
|
coverage=0.5,
|
|
longest_span=200,
|
|
legal_tags=["reproduction"],
|
|
evidence=[{"source_document_id": "doc-1", "excerpts": ["일치 문장"]}],
|
|
)
|
|
assert result.judgment_method == "llm"
|
|
assert result.llm_verdict == "likely"
|
|
assert result.llm_confidence == 0.82
|
|
assert result.llm_matched_precedent_ids == ("case-1",)
|
|
assert "case-1 판례" in result.judgment_summary
|
|
assert "저작권 침해가 의심" in result.judgment_summary
|
|
assert "보호되는 창작적 표현인지에 대한 사람 검토" in result.missing_factors
|
|
assert "시장 영향" in result.missing_factors
|
|
assert judge.request.evidence[0]["source_document_id"] == "doc-1"
|
|
|
|
|
|
def test_llm_hallucinated_precedent_id_falls_back_to_rules():
|
|
result = LegalRiskEngine([_case()], judge=_FakeJudge(("invented-case",))).assess(
|
|
max_similarity=0.9,
|
|
coverage=0.5,
|
|
longest_span=200,
|
|
legal_tags=["reproduction"],
|
|
)
|
|
assert result.judgment_method == "rule_fallback"
|
|
assert result.llm_verdict is None
|
|
assert result.precedent_ids == ("case-1",)
|
|
|
|
|
|
def test_llm_cannot_disable_human_review():
|
|
result = LegalRiskEngine([_case()], judge=_FakeJudge(review_required=False)).assess(
|
|
max_similarity=0.9,
|
|
coverage=0.5,
|
|
longest_span=200,
|
|
legal_tags=["reproduction"],
|
|
)
|
|
assert result.judgment_method == "rule_fallback"
|
|
assert result.llm_review_required is None
|
|
|
|
|
|
def test_likely_judgment_without_precedent_id_falls_back():
|
|
result = LegalRiskEngine([_case()], judge=_FakeJudge(())).assess(
|
|
max_similarity=0.9,
|
|
coverage=0.5,
|
|
longest_span=200,
|
|
legal_tags=["reproduction"],
|
|
)
|
|
assert result.judgment_method == "rule_fallback"
|
|
|
|
|
|
def test_openai_judge_uses_strict_schema_and_disables_response_storage():
|
|
captured = {}
|
|
|
|
class _Responses:
|
|
def create(self, **kwargs):
|
|
captured.update(kwargs)
|
|
return SimpleNamespace(output_text=json.dumps({
|
|
"verdict": "insufficient_evidence",
|
|
"confidence": 0.35,
|
|
"matched_precedent_ids": ["case-1"],
|
|
"supporting_reasons": [],
|
|
"counter_reasons": ["법적 맥락 부족"],
|
|
"missing_factors": ["의거관계"],
|
|
"review_required": True,
|
|
}, ensure_ascii=False))
|
|
|
|
client = SimpleNamespace(responses=_Responses())
|
|
fake = _FakeJudge()
|
|
engine = LegalRiskEngine([_case()], judge=fake)
|
|
engine.assess(
|
|
max_similarity=0.9, coverage=0.5, longest_span=200,
|
|
legal_tags=["reproduction"],
|
|
)
|
|
judge = OpenAILegalJudge(api_key="unused", model="test-gpt", client=client)
|
|
decision = judge.judge(fake.request)
|
|
|
|
assert decision.verdict == "insufficient_evidence"
|
|
assert captured["store"] is False
|
|
assert captured["text"]["format"]["type"] == "json_schema"
|
|
assert captured["text"]["format"]["strict"] is True
|