Mina Choi
11d30bb3d1
여러 줄 주석이 설명보다 경위(예전·실측·지적)를 적고 있어 읽는 사람이 결론을 찾기 어려웠다. - ts·tsx·js·mjs·css·py 478개: 여러 줄 주석은 첫 문장 한 줄로, 과거형·날짜 문장은 삭제 - 주석 위치는 TypeScript 파서·파이썬 tokenize/ast 로 찾는다 — 문자열 안의 # · /* 는 건드리지 않는다 - eslint·ts·noqa·type: ignore 같은 지시 주석은 그대로 둔다 파이썬 275개 정리 전후 AST 동일, TS 298개 주석 뺀 토큰 동일(빈 JSX 주석 10곳만 차이). site·frontend·admin tsc, site vitest 105 passed Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
220 lines
10 KiB
Python
220 lines
10 KiB
Python
"""수집 파이프라인 e2e — 잡이 실제로 돌아 fact·사진이 후보로 쌓이는지."""
|
|
import uuid
|
|
|
|
from common.enums import FactStatus, JobStatus, JobType, LinkChannel, MediaStatus, PlaceCategory, SourceType
|
|
from crud.job_crud import JobQueue
|
|
from services.collector import MockAdapter
|
|
from worker.handlers import build_handler
|
|
from worker.runner import Worker
|
|
|
|
|
|
async def _ready_place(client, h, category=PlaceCategory.LODGING, kakao="k1"):
|
|
"""검증까지 끝나고 MockAdapter 가 처리할 수 있는 링크가 확정된 사업장."""
|
|
pid = (await client.post("/v1/place", headers=h, json={"name": "하조대펜션", "category": category.value})).json()["place"]["place_id"]
|
|
await client.post(f"/v1/place/{pid}/verify", headers=h, json={
|
|
"external_place_id": kakao, "road_address": "강원 양양군 현북면 하조대해안길 3", "region_code": "4283025"})
|
|
url = MockAdapter.url_for(category, pid, channel="yanolja")
|
|
lid = (await client.post(f"/v1/place/{pid}/link", headers=h, json={
|
|
"channel": LinkChannel.YANOLJA.value, "url": url, "discovered_by": SourceType.API.value})).json()["link"]["link_id"]
|
|
await client.post(f"/v1/place/{pid}/link/{lid}/confirm", headers=h)
|
|
return pid
|
|
|
|
|
|
async def _run_worker(job_id=None):
|
|
"""워커를 돌려 큐를 비운다."""
|
|
worker = Worker("test-worker", JobQueue(), build_handler(), job_deadline_sec=60)
|
|
ran = 0
|
|
for _ in range(10):
|
|
if not await worker.process_one():
|
|
break
|
|
ran += 1
|
|
assert ran, "워커가 집을 잡이 없다"
|
|
|
|
|
|
async def test_pipeline_publishes_collected_facts(auth_headers, client):
|
|
"""검증: 수집 잡을 끝까지 돌린다."""
|
|
h = await auth_headers("u1")
|
|
pid = await _ready_place(client, h)
|
|
|
|
job_id = (await client.post(f"/v1/place/{pid}/collect", headers=h, json={})).json()["job_id"]
|
|
await _run_worker()
|
|
|
|
job = (await client.get(f"/v1/job/{job_id}", headers=h)).json()["job"]
|
|
assert job["status"] == JobStatus.DONE.value, job.get("last_error")
|
|
assert job["result"]["facts"]["stored"] > 0
|
|
|
|
listed = (await client.get(f"/v1/place/{pid}/fact/list", headers=h)).json()
|
|
assert len(listed["facts"]) > 0
|
|
assert listed["publishable"] > 0, "수집값이 노출값 자리에 들어가지 않았다"
|
|
assert all(
|
|
f["status"] in (FactStatus.VERIFIED.value, FactStatus.UNVERIFIED.value)
|
|
for f in listed["facts"]
|
|
)
|
|
|
|
|
|
async def test_pipeline_records_source_on_every_fact(auth_headers, client):
|
|
"""검증: 수집된 fact 의 출처."""
|
|
h = await auth_headers("u1")
|
|
pid = await _ready_place(client, h, kakao="k2")
|
|
await client.post(f"/v1/place/{pid}/collect", headers=h, json={})
|
|
await _run_worker()
|
|
|
|
facts = (await client.get(f"/v1/place/{pid}/fact/list", headers=h)).json()["facts"]
|
|
for f in facts:
|
|
assert f["source_type"] == SourceType.CRAWL.value
|
|
assert f["source_url"], f"출처 없는 fact: {f['key']}"
|
|
|
|
|
|
async def test_pipeline_creates_units_and_unit_scoped_facts(auth_headers, client):
|
|
"""검증: 숙박 수집 결과의 객실 단위 fact."""
|
|
h = await auth_headers("u1")
|
|
pid = await _ready_place(client, h, kakao="k3")
|
|
await client.post(f"/v1/place/{pid}/collect", headers=h, json={})
|
|
await _run_worker()
|
|
|
|
units = (await client.get(f"/v1/place/{pid}/unit/list", headers=h)).json()["units"]
|
|
assert len(units) >= 2
|
|
|
|
facts = (await client.get(f"/v1/place/{pid}/fact/list", headers=h)).json()["facts"]
|
|
unit_facts = [f for f in facts if f.get("unit_id")]
|
|
assert unit_facts, "객실 단위 fact 가 하나도 없다"
|
|
capacities = [f for f in unit_facts if f["key"] == "standard_capacity"]
|
|
assert len({f["unit_id"] for f in capacities}) >= 2, "객실별로 따로 붙어야 한다"
|
|
|
|
|
|
async def test_pipeline_stores_media_with_origin_and_pending_review(auth_headers, client):
|
|
"""검증: 수집된 사진."""
|
|
from sqlalchemy import text
|
|
|
|
h = await auth_headers("u1")
|
|
pid = await _ready_place(client, h, kakao="k4")
|
|
job_id = (await client.post(f"/v1/place/{pid}/collect", headers=h, json={})).json()["job_id"]
|
|
await _run_worker()
|
|
|
|
job = (await client.get(f"/v1/job/{job_id}", headers=h)).json()["job"]
|
|
assert job["result"]["media"]["stored"] > 0
|
|
|
|
|
|
async def test_recollect_skips_crawl_when_already_enough(auth_headers, client):
|
|
"""검증: 필수 항목이 이미 다 찬 사업장에 다시 수집을 건다."""
|
|
h = await auth_headers("u1")
|
|
pid = await _ready_place(client, h, kakao="k5")
|
|
|
|
await client.post(f"/v1/place/{pid}/collect", headers=h, json={})
|
|
await _run_worker()
|
|
first = (await client.get(f"/v1/place/{pid}/fact/list", headers=h)).json()
|
|
|
|
job_id = (await client.post(f"/v1/place/{pid}/collect", headers=h, json={})).json()["job_id"]
|
|
await _run_worker()
|
|
job = (await client.get(f"/v1/job/{job_id}", headers=h)).json()["job"]
|
|
|
|
assert job["result"]["coverage"]["enough"] is True
|
|
assert "크롤링 생략" in job["result"].get("note", "")
|
|
assert "fetch" not in job["result"], "충분한데 크롤링을 시도했다"
|
|
|
|
second = (await client.get(f"/v1/place/{pid}/fact/list", headers=h)).json()
|
|
assert len(second["facts"]) == len(first["facts"]), "재수집이 fact 를 중복 생성했다"
|
|
|
|
|
|
async def test_forced_recollect_is_idempotent(auth_headers, client):
|
|
"""검증: force=true 로 강제 재수집한다(항목이 이미 차 있어도)."""
|
|
h = await auth_headers("u1")
|
|
pid = await _ready_place(client, h, kakao="k5f")
|
|
|
|
await client.post(f"/v1/place/{pid}/collect", headers=h, json={})
|
|
await _run_worker()
|
|
first = (await client.get(f"/v1/place/{pid}/fact/list", headers=h)).json()
|
|
|
|
job_id = (await client.post(f"/v1/place/{pid}/collect", headers=h, json={"force": True})).json()["job_id"]
|
|
await _run_worker()
|
|
job = (await client.get(f"/v1/job/{job_id}", headers=h)).json()["job"]
|
|
|
|
assert job["result"]["fetch"]["fetched"] == 1, "force 인데 크롤링을 안 했다"
|
|
assert job["result"]["media"]["stored"] == 0
|
|
assert job["result"]["media"]["skipped_duplicate"] > 0
|
|
|
|
second = (await client.get(f"/v1/place/{pid}/fact/list", headers=h)).json()
|
|
assert len(second["facts"]) == len(first["facts"]), "재수집이 fact 를 중복 생성했다"
|
|
|
|
|
|
async def test_coverage_reports_missing_required_fields(auth_headers, client):
|
|
"""검증: 수집 후 필수 항목 충족도."""
|
|
h = await auth_headers("u1")
|
|
pid = await _ready_place(client, h, kakao="k5c")
|
|
|
|
job_id = (await client.post(f"/v1/place/{pid}/collect", headers=h, json={})).json()["job_id"]
|
|
await _run_worker()
|
|
cov = (await client.get(f"/v1/job/{job_id}", headers=h)).json()["job"]["result"]["coverage"]
|
|
|
|
assert cov["total"] > 0
|
|
assert cov["covered"] == cov["total"]
|
|
assert cov["missing"] == []
|
|
assert "check_in_time" in cov["required"]
|
|
|
|
|
|
async def test_recollect_does_not_touch_verified_value(auth_headers, client):
|
|
"""검증: 수집을 두 번 돌린다(같은 목데이터라 값이 같다)."""
|
|
h = await auth_headers("u1")
|
|
pid = await _ready_place(client, h, kakao="k6")
|
|
await client.post(f"/v1/place/{pid}/collect", headers=h, json={})
|
|
await _run_worker()
|
|
|
|
before = (await client.get(f"/v1/place/{pid}/fact/list", headers=h)).json()
|
|
assert before["publishable"] > 0
|
|
check_in_before = [f for f in before["facts"] if f["key"] == "check_in_time"]
|
|
assert len(check_in_before) == 1 and check_in_before[0]["status"] == FactStatus.VERIFIED.value
|
|
|
|
await client.post(f"/v1/place/{pid}/collect", headers=h, json={})
|
|
await _run_worker()
|
|
|
|
after = (await client.get(f"/v1/place/{pid}/fact/list", headers=h)).json()
|
|
assert after["publishable"] == before["publishable"], "★ 재수집이 노출값 수를 바꿨다"
|
|
check_in = [f for f in after["facts"] if f["key"] == "check_in_time"]
|
|
assert len(check_in) == 1 and check_in[0]["status"] == FactStatus.VERIFIED.value
|
|
|
|
|
|
async def test_recollect_cannot_overwrite_corrected_value(auth_headers, client):
|
|
"""검증: 사장님이 정정한 값에 재수집이 다른 값을 들고 온다."""
|
|
h = await auth_headers("u1")
|
|
pid = await _ready_place(client, h, kakao="k7")
|
|
await client.post(f"/v1/place/{pid}/collect", headers=h, json={})
|
|
await _run_worker()
|
|
|
|
facts = (await client.get(f"/v1/place/{pid}/fact/list", headers=h)).json()["facts"]
|
|
target = next(f for f in facts if f["key"] == "check_in_time")
|
|
fid = target["fact_id"]
|
|
await client.post(f"/v1/place/{pid}/fact/{fid}/transition", headers=h, json={"status": FactStatus.VERIFIED.value})
|
|
await client.post(f"/v1/place/{pid}/fact/{fid}/transition", headers=h,
|
|
json={"status": FactStatus.CORRECTED.value, "value": "16:30"})
|
|
|
|
await client.post(f"/v1/place/{pid}/collect", headers=h, json={})
|
|
await _run_worker()
|
|
|
|
published = (await client.get(f"/v1/place/{pid}/fact/list", headers=h, params={"publishable_only": True})).json()["facts"]
|
|
check_in = [f for f in published if f["key"] == "check_in_time"]
|
|
assert len(check_in) == 1
|
|
assert check_in[0]["value"] == "16:30", "★ 자동 수집이 사장님 정정본을 덮어썼다"
|
|
assert check_in[0]["status"] == FactStatus.CORRECTED.value
|
|
|
|
|
|
async def test_pipeline_refuses_unverified_place(db_engine, owner_id):
|
|
"""검증: 검증 안 된 사업장의 수집 잡이 큐에 직접 들어간 경우(잡 적재 후 검증이 취소된 상황)."""
|
|
from sqlalchemy import text
|
|
|
|
pid = uuid.uuid4()
|
|
async with db_engine.begin() as conn:
|
|
await conn.execute(
|
|
text("INSERT INTO places (place_id, owner_user_id, name, category, status) "
|
|
"VALUES (:pid, :cid, :n, 1, 1)"),
|
|
{"pid": pid, "cid": uuid.UUID(owner_id), "n": "미검증펜션"},
|
|
)
|
|
|
|
q = JobQueue()
|
|
job_id = await q.enqueue(JobType.COLLECT.value, {"place_id": str(pid), "owner_user_id": owner_id}, max_attempts=1)
|
|
worker = Worker("test-worker", q, build_handler(), backoff_fn=lambda _a: 0)
|
|
await worker.process_one()
|
|
|
|
row = await q.get(job_id)
|
|
assert row["status"] == JobStatus.DEAD.value
|
|
assert "동일 업소 검증" in row["last_error"]
|