From 4b8dee8583f34b5ced993053a4deda9f56a16530 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=EB=AF=BC=ED=97=8C?= Date: Thu, 9 Jul 2026 17:01:09 +0900 Subject: [PATCH] =?UTF-8?q?perf(lps):=20Step1=20=E2=80=94=20=ED=8F=B4?= =?UTF-8?q?=EB=B0=B1=20=EB=8D=B0=EB=93=9C=EB=9D=BC=EC=9D=B8=20+=20fail-fas?= =?UTF-8?q?t=EB=A1=9C=20=EA=B2=80=EC=83=89=20=EC=A7=80=EC=97=B0=20?= =?UTF-8?q?=EC=83=81=ED=95=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit G마켓 챌린지가 안 풀린 채 25~70초를 먹던 문제에 상한을 씌운다. 폴백은 '있으면 좋은' 보강이라 데드라인으로 잘라도 안전(그 몰만 스킵). - handler: 각 폴백 크롤에 fallback_deadline_sec(기본 15s) wait_for. 초과 시 스킵. _timed_search 예외를 BaseException 로(CancelledError 계측 포함). - 폴백 어댑터 max_block_retries=0(챌린지 대기 2배 방지), ESM ready_timeout 22→12s, st11 20→15s. - 테스트: 느린 폴백 데드라인 스킵 1종. 실측(동시성2·4상품): 상품당 p50 84.6s→34.2s(~2.5x), 벽시계 181s→65s(~2.8x), 처리량 1.3→3.7 상품/분, 1000건 $7.12→$5.06(~29%↓, 실패 챌린지 대역폭 절감). Co-Authored-By: Claude Fable 5 --- lps/services/search/esm/adapter.py | 2 +- lps/services/search/st11/adapter.py | 2 +- lps/tests/test_search_handler.py | 23 ++++++++++++++++++++++- lps/worker/handlers.py | 14 +++++++++++--- lps/worker_main.py | 7 ++++--- 5 files changed, 39 insertions(+), 9 deletions(-) diff --git a/lps/services/search/esm/adapter.py b/lps/services/search/esm/adapter.py index 8e0e0ec..beb8938 100644 --- a/lps/services/search/esm/adapter.py +++ b/lps/services/search/esm/adapter.py @@ -20,7 +20,7 @@ _BLOCK_MARKERS = ("errors.edgesuite.net", "You don't have permission to access", class EsmAdapter(BrowserSearchAdapter): block_markers = _BLOCK_MARKERS min_result_html = 8000 # 챌린지 페이지(~21KB)는 크므로 short 폴백은 명백한 에러만 - ready_timeout_ms = 22000 # '잠시만' 챌린지 통과(~9초) 여유 + ready_timeout_ms = 12000 # 챌린지 통과(정상 ~9초). 폴백은 데드라인이 상한이라 과도한 대기 회피 block_resources_default = True # 라우팅 on — 단, 아래 집합만(CSS/JS 유지) blocked_resource_types = {"image", "media", "font"} # 대역폭 대부분(이미지)만 차단, 챌린지/렌더용 CSS/JS는 유지 diff --git a/lps/services/search/st11/adapter.py b/lps/services/search/st11/adapter.py index 4c26d70..f31ccfe 100644 --- a/lps/services/search/st11/adapter.py +++ b/lps/services/search/st11/adapter.py @@ -19,7 +19,7 @@ class ElevenStAdapter(BrowserSearchAdapter): block_markers = _BLOCK_MARKERS min_result_html = 20000 # PC 정상 결과는 수백 KB — 차단/미렌더 페이지는 작다(→회전/재시도) ready_selector = READY_SELECTOR - ready_timeout_ms = 20000 + ready_timeout_ms = 15000 block_resources_default = True # 라우팅 on — 이미지/미디어/폰트만(CSS/JS 유지해야 렌더됨) blocked_resource_types = {"image", "media", "font"} scroll_steps = 3 # 결과가 지연 로딩 → 스크롤로 트리거 diff --git a/lps/tests/test_search_handler.py b/lps/tests/test_search_handler.py index dfec092..d4f52f7 100644 --- a/lps/tests/test_search_handler.py +++ b/lps/tests/test_search_handler.py @@ -1,5 +1,7 @@ """검색 핸들러 테스트 — 병합·실패격리·AI판정·재정제 루프·not_found·네거티브 캐시 (fake 의존성).""" +import asyncio + import pytest from common.enums import JobType @@ -8,17 +10,20 @@ from worker.handlers import build_search_handler class FakeAdapter: - def __init__(self, source, by_query=None, products=None, fail=False, uses_proxy=False, last_bytes=0): + def __init__(self, source, by_query=None, products=None, fail=False, uses_proxy=False, last_bytes=0, delay=0.0): self.source = source self._by_query = by_query # {query: [products]} self._products = products or [] self._fail = fail + self._delay = delay # search 지연(초) — 데드라인 테스트용 self.uses_proxy = uses_proxy # DECODO 경유 여부(비용 귀속) self.last_bytes = last_bytes self.calls = [] async def search(self, query, limit=40): self.calls.append(query) + if self._delay: + await asyncio.sleep(self._delay) if self._fail: raise AdapterError("boom", source=self.source, blocked=True) if self._by_query is not None: @@ -150,6 +155,22 @@ async def test_fallback_crawls_only_uncovered_malls(): assert malls == {"G마켓", "옥션", "11번가"} # 네이버 G마켓 + 크롤 옥션·11번가 +async def test_fallback_deadline_skips_slow_mall(): + # 느린 폴백(데드라인 초과)은 스킵되고, 빠른 폴백은 병합된다 — 전체 지연에 상한. + adapters = {"naver": FakeAdapter("naver", products=[_np("naver", 9000, mall="네이버")])} + slow = FakeAdapter("gmarket", products=[_np("gmarket", 1000, mall="G마켓")], delay=1.0) # 데드라인 초과 + fast = FakeAdapter("st11", products=[_np("st11", 3000, mall="11번가")], delay=0.0) + r = await build_search_handler( + adapters, judge=FakeJudge(lambda c: True), + fallback_adapters={"gmarket": slow, "st11": fast}, + fallback_deadline_sec=0.2, + )(_job()) + malls = {m["mall_name"] for m in r["by_mall"]} + assert "11번가" in malls # 빠른 폴백 병합됨 + assert "G마켓" not in malls # 느린 폴백은 데드라인 초과로 스킵 + assert r["lowest"]["price"] == 3000 # G마켓 1000은 스킵됐으므로 최저가 아님 + + async def test_fallback_failure_is_isolated(): adapters = {"naver": FakeAdapter("naver", products=[_np("naver", 9000, mall="네이버")])} st11 = FakeAdapter("st11", fail=True) # 크롤 실패 diff --git a/lps/worker/handlers.py b/lps/worker/handlers.py index 17b9bc9..9b0ba8b 100644 --- a/lps/worker/handlers.py +++ b/lps/worker/handlers.py @@ -56,6 +56,7 @@ def build_search_handler( fallback_adapters: dict[str, SearchAdapter] | None = None, ai_model: str = "", proxy_cost_per_gb: float = 0.0, + fallback_deadline_sec: float = 15.0, ): """검색 핸들러 생성. judge: SimilarityJudge(같은 상품 판정) / keyword_gen: KeywordGenerator(정밀·광역 재검색어) / @@ -82,8 +83,8 @@ def build_search_handler( res = await adapter.search(query, limit=limit) metrics.add_fetch(source, getattr(adapter, "last_bytes", 0), int((time.monotonic() - t0) * 1000), crawl=crawl, via_proxy=via_proxy) return res - except Exception: - metrics.add_fetch(source, 0, int((time.monotonic() - t0) * 1000), crawl=crawl, via_proxy=via_proxy) + except BaseException: # CancelledError(데드라인 취소) 포함 — 소요/바이트는 계측하고 재전파 + metrics.add_fetch(source, getattr(adapter, "last_bytes", 0), int((time.monotonic() - t0) * 1000), crawl=crawl, via_proxy=via_proxy) raise async def _search_round(query: str, metrics: SearchMetrics): @@ -124,8 +125,15 @@ def build_search_handler( return matched async def _crawl_match(src, adapter): + # 폴백은 '있으면 좋은' 보강이라 데드라인을 건다 — 초과 시 그 몰만 스킵(전체 지연에 상한). try: - crawled = await _timed_search(adapter, query, src, metrics, crawl=True) + crawled = await asyncio.wait_for( + _timed_search(adapter, query, src, metrics, crawl=True), + timeout=fallback_deadline_sec, + ) + except asyncio.TimeoutError: + LOG.w(f"[fallback:{src}] 데드라인 {fallback_deadline_sec:.0f}s 초과 → 스킵") + return [] except Exception as ex: LOG.w(f"[fallback:{src}] 크롤 실패(무시): {type(ex).__name__}: {ex}") return [] diff --git a/lps/worker_main.py b/lps/worker_main.py index ebcbb39..a286e6e 100644 --- a/lps/worker_main.py +++ b/lps/worker_main.py @@ -46,10 +46,11 @@ def _build_worker(i: int, concurrency: int, has_openai: bool, neg_cache, history "coupang": CoupangAdapter(headless=False, user_data_dir=_pf("coupang"), proxy=proxy, on_detect=bot_log.record), "naver": NaverAdapter(), # httpx 직접(프록시 미경유) — 워커별 인스턴스(last_bytes 경합 회피) } + # 폴백은 데드라인이 상한이라 봇감지 재시도(챌린지 대기 2배)를 끈다(max_block_retries=0) — 빠르게 포기·스킵. fallback_adapters = { - "gmarket": EsmAdapter("gmarket", headless=False, user_data_dir=_pf("gmarket"), proxy=proxy, on_detect=bot_log.record), - "auction": EsmAdapter("auction", headless=False, user_data_dir=_pf("auction"), proxy=proxy, on_detect=bot_log.record), - "st11": ElevenStAdapter(headless=False, user_data_dir=_pf("st11"), proxy=proxy, on_detect=bot_log.record), + "gmarket": EsmAdapter("gmarket", headless=False, user_data_dir=_pf("gmarket"), proxy=proxy, on_detect=bot_log.record, max_block_retries=0), + "auction": EsmAdapter("auction", headless=False, user_data_dir=_pf("auction"), proxy=proxy, on_detect=bot_log.record, max_block_retries=0), + "st11": ElevenStAdapter(headless=False, user_data_dir=_pf("st11"), proxy=proxy, on_detect=bot_log.record, max_block_retries=0), } # AI 도 워커별 인스턴스 — 공유 상태(last_usage) 경합 원천 제거 judge = SimilarityJudge() if has_openai else None