84 lines
3.6 KiB
Python
84 lines
3.6 KiB
Python
"""① fetch — NOL 티켓(인터파크) 상품 페이지에서 포스터·상세 이미지·메타를 수집한다.
|
|
|
|
상품 페이지가 Next.js SSR이라 첫 HTML 안에 상품 JSON과 상세 소개 HTML이 통째로 들어 있어
|
|
브라우저 없이 정적으로 뽑는다.
|
|
|
|
meta를 못 읽으면 상품페이지 구조가 바뀌었거나 goodsId가 틀렸다는 뜻이라 즉시 실패한다.
|
|
"""
|
|
import asyncio
|
|
import re
|
|
|
|
import httpx
|
|
|
|
from models.nol import FetchedImage, NolMeta, NolProduct
|
|
from utils.image import sniff_extension
|
|
|
|
PRODUCT_URL = "https://nol.yanolja.com/ticket/products/{goods_id}"
|
|
USER_AGENT = ("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/120 Safari/537.36")
|
|
|
|
# 상품 JSON에서 뽑을 키. NolMeta의 alias와 같아야 한다.
|
|
META_KEYS = ("goodsName", "placeName", "playStartDate", "playEndDate", "runningTime",
|
|
"interMissionTime", "viewRateName", "genreName", "subGenreName",
|
|
"corporationName", "posterImageUrl", "bookingOpenTime")
|
|
|
|
# 배우 프로필 썸네일·공연장 사진은 영상 소재가 아니다
|
|
SKIP_PATTERN = re.compile(r"PlayDictionary|Play/Place/|favicon|yaimg\.yanolja")
|
|
POSTER_PATH = "/Play/image/large/"
|
|
DETAIL_IMG_PATTERN = re.compile(
|
|
r'<img[^>]+src="(https?://ticketimage\.interpark\.com/[^"]+)"')
|
|
|
|
def parse_product(html: str, goods_id: str) -> NolMeta:
|
|
unescaped = html.replace("\\u0026", "&").replace('\\"', '"').replace("\\/", "/")
|
|
raw: dict = {}
|
|
for key in META_KEYS:
|
|
found = re.search(key + r'":"([^"]*)"', unescaped)
|
|
raw[key] = found.group(1) if found else None
|
|
price = re.search(r'"price":(\d+)', unescaped)
|
|
|
|
seen: set[str] = set()
|
|
detail_urls = []
|
|
for url in DETAIL_IMG_PATTERN.findall(unescaped): # 등장 순서 유지, 중복 제거
|
|
if SKIP_PATTERN.search(url) or POSTER_PATH in url or url in seen:
|
|
continue
|
|
seen.add(url)
|
|
detail_urls.append(url.replace("http://", "https://"))
|
|
|
|
return NolMeta(goods_id=goods_id, url=PRODUCT_URL.format(goods_id=goods_id),
|
|
min_price=int(price.group(1)) if price else None,
|
|
detail_image_urls=detail_urls, **raw)
|
|
|
|
|
|
async def download(client: httpx.AsyncClient, url: str) -> FetchedImage | None:
|
|
try:
|
|
response = await client.get(url)
|
|
response.raise_for_status()
|
|
except httpx.HTTPError:
|
|
return None
|
|
if not response.content:
|
|
return None
|
|
return FetchedImage(url=url, data=response.content,
|
|
extension=sniff_extension(response.content))
|
|
|
|
|
|
async def fetch_product(goods_id: str) -> NolProduct:
|
|
async with httpx.AsyncClient(timeout=90, follow_redirects=True,
|
|
headers={"User-Agent": USER_AGENT}) as client:
|
|
page = await client.get(PRODUCT_URL.format(goods_id=goods_id))
|
|
page.raise_for_status()
|
|
meta = parse_product(page.text, goods_id)
|
|
if not meta.goods_name:
|
|
raise RuntimeError(f"상품 정보를 못 읽었다 (goodsId={goods_id}) — "
|
|
"페이지 구조가 바뀌었거나 id가 틀렸다")
|
|
|
|
poster = (await download(client, meta.poster_image_url)
|
|
if meta.poster_image_url else None)
|
|
details = await asyncio.gather(
|
|
*(download(client, url) for url in meta.detail_image_urls))
|
|
|
|
return NolProduct(
|
|
meta=meta, poster=poster,
|
|
details=[image for image in details if image],
|
|
failed_urls=[url for url, image in zip(meta.detail_image_urls, details, strict=True)
|
|
if image is None])
|