[fix] solution/backend: 네이버 플레이스 수집이 음식점·카페 요금표를 통째로 반려 — 업종별 unit key 분기

_to_unit_facts() 가 업종을 안 가리고 room_type·weekday_price/weekend_price 로만 냈다.
그 key 는 숙박 스키마에만 있어서, 음식점·카페·클리닉은 메뉴/프로그램 요금표가
FACT_INVALID_KEY 로 전량 거부됐다(실측: "도플로" fact 19건 중 19건 반려).

- naver_place_adapter.py: _UNIT_NAME_KEY·_UNIT_PRICE_KEY 로 업종별 key 매핑
  (숙박 room_type/weekday·weekend_price, 카페·음식점 menu_name/menu_price,
  클리닉 program_name/price_adult)
- SourceAdapter.fetch() 계약에 category 파라미터 추가, 어댑터 5개 시그니처 반영
- collect_service.py: fetch_one() 에 place.category 전달

검증: 재수집 후 stored 0 → 35(음식점), test_collector·test_category_schema·
test_fact_api·test_tour_api_adapter 173 passed
This commit is contained in:
Mina Choi 2026-09-23 13:17:28 +09:00
parent c366361513
commit 951e451ef8
7 changed files with 38 additions and 21 deletions

View File

@ -412,7 +412,7 @@ async def coverage(place, place_id: str) -> dict:
} }
async def fetch_one(link): async def fetch_one(link, category=None):
"""링크 하나를 긁는다. 실패해도 예외를 던지지 않는다 — 나머지 링크가 살아야 한다.""" """링크 하나를 긁는다. 실패해도 예외를 던지지 않는다 — 나머지 링크가 살아야 한다."""
try: try:
adapter = REGISTRY.get_adapter(link.url) adapter = REGISTRY.get_adapter(link.url)
@ -421,7 +421,7 @@ async def fetch_one(link):
LOG.w(f"[collect] 어댑터 없음 — 건너뜀 {link.url}: {type(ex).__name__}") LOG.w(f"[collect] 어댑터 없음 — 건너뜀 {link.url}: {type(ex).__name__}")
return None, "no_adapter" return None, "no_adapter"
try: try:
source = await adapter.fetch(link.url) source = await adapter.fetch(link.url, category)
except Exception as ex: except Exception as ex:
collect_diagnostics.note_issue("fetch", link.url, ex) collect_diagnostics.note_issue("fetch", link.url, ex)
return None, "failed" return None, "failed"
@ -638,7 +638,7 @@ async def _run_collect(job: dict) -> dict:
f"남은 링크 {fetch_stat['skipped_enough']}건 크롤링 생략") f"남은 링크 {fetch_stat['skipped_enough']}건 크롤링 생략")
break break
source, outcome = await fetch_one(link) source, outcome = await fetch_one(link, PlaceCategory(place.category))
fetch_stat[outcome] += 1 fetch_stat[outcome] += 1
if source is None: if source is None:
continue continue

View File

@ -11,7 +11,7 @@ from dataclasses import dataclass, field
from datetime import datetime, timezone from datetime import datetime, timezone
from typing import Optional, Protocol, runtime_checkable from typing import Optional, Protocol, runtime_checkable
from common.enums import LinkChannel from common.enums import LinkChannel, PlaceCategory
# ---- 도메인 예외 ----------------------------------------------------------- # ---- 도메인 예외 -----------------------------------------------------------
@ -154,4 +154,4 @@ class SourceAdapter(Protocol):
def can_handle(self, url: str) -> bool: ... def can_handle(self, url: str) -> bool: ...
async def fetch(self, url: str) -> RawSource: ... async def fetch(self, url: str, category: Optional[PlaceCategory] = None) -> RawSource: ...

View File

@ -14,6 +14,7 @@ URL 규약 (업종을 URL 에서 읽어 결정적으로 동작한다)
mock://cafe/cafe-1?channel=naver_place mock://cafe/cafe-1?channel=naver_place
https://mock.test/restaurant/r-1 https://mock.test/restaurant/r-1
""" """
from typing import Optional
from urllib.parse import parse_qs, urlparse from urllib.parse import parse_qs, urlparse
from common.category_schema import get_schema from common.category_schema import get_schema
@ -227,7 +228,7 @@ class MockAdapter:
return True return True
return parsed.scheme in ("http", "https") and parsed.hostname in _HOSTS return parsed.scheme in ("http", "https") and parsed.hostname in _HOSTS
async def fetch(self, url: str) -> RawSource: async def fetch(self, url: str, category: Optional[PlaceCategory] = None) -> RawSource:
"""URL 에서 업종을 읽어 그 업종의 목데이터를 돌려준다. """URL 에서 업종을 읽어 그 업종의 목데이터를 돌려준다.
업종을 못 읽으면 예외가 아니라 실패 결과(ok=False)로 돌려준다 — 업종을 못 읽으면 예외가 아니라 실패 결과(ok=False)로 돌려준다 —

View File

@ -21,7 +21,7 @@ from typing import Optional
import httpx import httpx
from common.enums import LinkChannel from common.enums import LinkChannel, PlaceCategory
from common.logger import LOG from common.logger import LOG
from services.collector.base import CollectedFact, CollectedMedia, RawSource from services.collector.base import CollectedFact, CollectedMedia, RawSource
@ -102,6 +102,18 @@ _WEEKEND_TOKENS = ("주말", "금토", "토일", "공휴일")
# 요일 토큰과, 바로 뒤에 붙는 괄호 보충설명("주말(금,토)")까지 한 번에 걷어낸다. # 요일 토큰과, 바로 뒤에 붙는 괄호 보충설명("주말(금,토)")까지 한 번에 걷어낸다.
_DAY_TAG = re.compile(rf"({'|'.join(_WEEKDAY_TOKENS + _WEEKEND_TOKENS)})\s*(\([^)]*\))?\s*") _DAY_TAG = re.compile(rf"({'|'.join(_WEEKDAY_TOKENS + _WEEKEND_TOKENS)})\s*(\([^)]*\))?\s*")
_UNIT_NAME_KEY = {
PlaceCategory.LODGING: "room_type",
PlaceCategory.CAFE: "menu_name",
PlaceCategory.RESTAURANT: "menu_name",
PlaceCategory.CLINIC: "program_name",
}
_UNIT_PRICE_KEY = {
PlaceCategory.CAFE: "menu_price",
PlaceCategory.RESTAURANT: "menu_price",
PlaceCategory.CLINIC: "price_adult",
}
class NaverPlaceAdapter: class NaverPlaceAdapter:
"""네이버 플레이스 상세 → fact·사진 후보.""" """네이버 플레이스 상세 → fact·사진 후보."""
@ -125,7 +137,7 @@ class NaverPlaceAdapter:
u = (url or "").lower() u = (url or "").lower()
return any(h in u for h in _HOSTS) return any(h in u for h in _HOSTS)
async def fetch(self, url: str) -> RawSource: async def fetch(self, url: str, category: Optional[PlaceCategory] = None) -> RawSource:
channel = LinkChannel.NAVER_PLACE channel = LinkChannel.NAVER_PLACE
try: try:
place_id = await self._resolve_place_id(url) place_id = await self._resolve_place_id(url)
@ -143,7 +155,7 @@ class NaverPlaceAdapter:
if not base: if not base:
return RawSource.failure(url, self.id, "응답에 PlaceDetailBase 가 없다", channel) return RawSource.failure(url, self.id, "응답에 PlaceDetailBase 가 없다", channel)
facts = self._to_facts(base, state) facts = self._to_facts(base, state, category)
media = self._to_media(state) media = self._to_media(state)
booking_url = self._booking_url(state) booking_url = self._booking_url(state)
@ -293,7 +305,7 @@ class NaverPlaceAdapter:
raise RuntimeError(f"네트워크 오류: {ex}") raise RuntimeError(f"네트워크 오류: {ex}")
raise RuntimeError(last) raise RuntimeError(last)
def _to_facts(self, base: dict, state: dict) -> list[CollectedFact]: def _to_facts(self, base: dict, state: dict, category: Optional[PlaceCategory] = None) -> list[CollectedFact]:
"""★ 스키마에 있는 key 만 만든다. 없는 key 는 fact 기록 단계에서 통째로 거부된다.""" """★ 스키마에 있는 key 만 만든다. 없는 key 는 fact 기록 단계에서 통째로 거부된다."""
facts: list[CollectedFact] = [] facts: list[CollectedFact] = []
@ -345,10 +357,10 @@ class NaverPlaceAdapter:
seen.add(key) seen.add(key)
facts.append(CollectedFact(key=key, value="false" if negated else "true")) facts.append(CollectedFact(key=key, value="false" if negated else "true"))
facts.extend(self._to_unit_facts(state)) facts.extend(self._to_unit_facts(state, category))
return facts return facts
def _to_unit_facts(self, state: dict) -> list[CollectedFact]: def _to_unit_facts(self, state: dict, category: Optional[PlaceCategory] = None) -> list[CollectedFact]:
"""요금표(`Menu:*`) → 단위(객실·메뉴·프로그램) 스코프 fact. """요금표(`Menu:*`) → 단위(객실·메뉴·프로그램) 스코프 fact.
★ 왜 필요한가 ★ 왜 필요한가
@ -368,6 +380,9 @@ class NaverPlaceAdapter:
rows = [v for k, v in state.items() if k.startswith("Menu") and isinstance(v, dict)] rows = [v for k, v in state.items() if k.startswith("Menu") and isinstance(v, dict)]
rows.sort(key=lambda v: int(v.get("index") or 0)) rows.sort(key=lambda v: int(v.get("index") or 0))
name_key = _UNIT_NAME_KEY.get(category, "room_type")
flat_price_key = _UNIT_PRICE_KEY.get(category) if category is not None else None
out: list[CollectedFact] = [] out: list[CollectedFact] = []
seen_names: list[str] = [] seen_names: list[str] = []
for row in rows: for row in rows:
@ -382,16 +397,17 @@ class NaverPlaceAdapter:
if unit_name not in seen_names: if unit_name not in seen_names:
seen_names.append(unit_name) seen_names.append(unit_name)
# room_type 은 숙박 스키마의 unit 필수 필드다. 이름 자체가 상품 구분이므로 그대로 싣는다.
out.append(CollectedFact( out.append(CollectedFact(
key="room_type", value=unit_name, scope="unit", unit_name=unit_name, key=name_key, value=unit_name, scope="unit", unit_name=unit_name,
)) ))
# 요금. 네이버는 문자열 숫자("20000")로 준다 — 표기는 렌더 단계(format_value)가 만든다. # 요금. 네이버는 문자열 숫자("20000")로 준다 — 표기는 렌더 단계(format_value)가 만든다.
price = str(row.get("price") or "").strip().replace(",", "") price = str(row.get("price") or "").strip().replace(",", "")
if not price.isdigit(): if not price.isdigit():
continue continue
if any(t in raw_name for t in _WEEKEND_TOKENS): if flat_price_key:
price_key = flat_price_key
elif any(t in raw_name for t in _WEEKEND_TOKENS):
price_key = "weekend_price" price_key = "weekend_price"
elif any(t in raw_name for t in _WEEKDAY_TOKENS): elif any(t in raw_name for t in _WEEKDAY_TOKENS):
price_key = "weekday_price" price_key = "weekday_price"

View File

@ -47,7 +47,7 @@ from urllib.robotparser import RobotFileParser
import httpx import httpx
from common.enums import LinkChannel from common.enums import LinkChannel, PlaceCategory
from common.logger import LOG from common.logger import LOG
from services.collector.base import CollectedFact, CollectedMedia, RawSource from services.collector.base import CollectedFact, CollectedMedia, RawSource
@ -200,7 +200,7 @@ class StaticHtmlAdapter:
return False return False
return not any(host == d or host.endswith("." + d) for d in _DENY_HOSTS) return not any(host == d or host.endswith("." + d) for d in _DENY_HOSTS)
async def fetch(self, url: str) -> RawSource: async def fetch(self, url: str, category: Optional[PlaceCategory] = None) -> RawSource:
channel = self._channel(url) channel = self._channel(url)
allowed, why = await self._robots_allows(url) allowed, why = await self._robots_allows(url)

View File

@ -33,7 +33,7 @@ from urllib.parse import unquote, urlencode
import httpx import httpx
from common.enums import LinkChannel from common.enums import LinkChannel, PlaceCategory
from common.logger import LOG from common.logger import LOG
from config.server_configs import external_api_config from config.server_configs import external_api_config
from services.collector.base import CollectedFact, CollectedMedia, RawSource from services.collector.base import CollectedFact, CollectedMedia, RawSource
@ -104,7 +104,7 @@ class TourApiAdapter:
return True return True
return any(h in u for h in _HOSTS) and bool(_COTID.search(u)) return any(h in u for h in _HOSTS) and bool(_COTID.search(u))
async def fetch(self, url: str) -> RawSource: async def fetch(self, url: str, category: Optional[PlaceCategory] = None) -> RawSource:
channel = LinkChannel.ETC channel = LinkChannel.ETC
key = (external_api_config.tour_api_key or "").strip() key = (external_api_config.tour_api_key or "").strip()
if not key: if not key:

View File

@ -23,7 +23,7 @@ from typing import Optional
from playwright.async_api import Page, TimeoutError as PWTimeoutError, async_playwright from playwright.async_api import Page, TimeoutError as PWTimeoutError, async_playwright
from common.enums import LinkChannel from common.enums import LinkChannel, PlaceCategory
from common.logger import LOG from common.logger import LOG
from services.collector.base import CollectedFact, CollectedMedia, RawSource from services.collector.base import CollectedFact, CollectedMedia, RawSource
@ -249,7 +249,7 @@ class YanoljaAdapter:
def can_handle(self, url: str) -> bool: def can_handle(self, url: str) -> bool:
return bool(DETAIL_URL_RE.search((url or "").lower())) return bool(DETAIL_URL_RE.search((url or "").lower()))
async def fetch(self, url: str) -> RawSource: async def fetch(self, url: str, category: Optional[PlaceCategory] = None) -> RawSource:
pw, browser, page = await _new_page() pw, browser, page = await _new_page()
try: try:
await page.goto(url, wait_until="domcontentloaded", timeout=60000) await page.goto(url, wait_until="domcontentloaded", timeout=60000)