castad
/
o2o-castad-backend
3
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

크롤링 디버그 로그 추가

get_video
dhlim 2026-02-04 01:05:59 +00:00
parent 89ea0c783e
commit f24ff46b09
1 changed files with 22 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
app/utils/nvMapPwScraper.py
View File

@ -1,6 +1,11 @@
import asyncio
from playwright.async_api import async_playwright
from urllib import parse
import time
from app.utils.logger import get_logger
# 로거 설정
logger = get_logger("pwscraper")
class NvMapPwScraper():
# cls vars
@ -91,25 +96,41 @@ patchedGetter.toString();''')
async def get_place_id_url(self, selected):
count = 0
get_place_id_url_start = time.perf_counter()
while (count <= 1):
title = selected['title'].replace("<b>", "").replace("</b>", "")
address = selected.get('roadAddress', selected['address']).replace("<b>", "").replace("</b>", "")
encoded_query = parse.quote(f"{address} {title}")
url = f"https://map.naver.com/p/search/{encoded_query}"
wait_first_start = time.perf_counter()
await self.goto_url(url, wait_until="networkidle",timeout = self._max_retry/2*1000)
wait_first_time = (time.perf_counter() - wait_first_start) * 1000
logger.debug(f"[DEBUG] Try {count+1} : Wait for perfect matching : {wait_first_time}ms")
if "/place/" in self.page.url:
return self.page.url
logger.debug(f"[DEBUG] Try {count+1} : url place id not found, retry for forced collect answer")
wait_forced_correct_start = time.perf_counter()
url = self.page.url.replace("?","?isCorrectAnswer=true&")
await self.goto_url(url, wait_until="networkidle",timeout = self._max_retry/2*1000)
wait_forced_correct_time = (time.perf_counter() - wait_forced_correct_start) * 1000
logger.debug(f"[DEBUG] Try {count+1} : Wait for forced isCorrectAnswer flag : {wait_forced_correct_time}ms")
if "/place/" in self.page.url:
return self.page.url
count += 1
print("Not found url for {selected}")
logger.error("[ERROR] Not found url for {selected}")
return None # 404