72 lines
3.4 KiB
Python
72 lines
3.4 KiB
Python
import logging
|
|
from common.db import (
|
|
set_instagram_status, save_instagram_raw_data,
|
|
set_facebook_status, save_facebook_raw_data,
|
|
set_naver_blog_status, save_naver_blog_raw_data,
|
|
set_youtube_status, save_youtube_raw_data,
|
|
set_gangnam_unni_status, save_gangnam_unni_raw_data,
|
|
execute, save_hospital_raw_data,
|
|
)
|
|
from common.utils import get_env
|
|
from services.analysis import check_and_advance_analysis
|
|
from integrations.apify import ApifyClient
|
|
from integrations.naver import NaverClient
|
|
from integrations.youtube import YouTubeClient
|
|
from integrations.firecrawl import FirecrawlClient
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
async def collect_instagram(analysis_run_id: str, row_id: int, url: str) -> None:
|
|
logger.info("[instagram] start run=%s url=%s", analysis_run_id, url)
|
|
await set_instagram_status(row_id, "processing")
|
|
data = await ApifyClient(get_env("APIFY_API_TOKEN")).get_instagram_profile(url)
|
|
await save_instagram_raw_data(row_id, data)
|
|
logger.info("[instagram] done run=%s", analysis_run_id)
|
|
await check_and_advance_analysis(analysis_run_id)
|
|
|
|
|
|
async def collect_facebook(analysis_run_id: str, row_id: int, url: str) -> None:
|
|
logger.info("[facebook] start run=%s url=%s", analysis_run_id, url)
|
|
await set_facebook_status(row_id, "processing")
|
|
data = await ApifyClient(get_env("APIFY_API_TOKEN")).get_facebook_page(url)
|
|
await save_facebook_raw_data(row_id, data)
|
|
logger.info("[facebook] done run=%s", analysis_run_id)
|
|
await check_and_advance_analysis(analysis_run_id)
|
|
|
|
|
|
async def collect_naver_blog(analysis_run_id: str, row_id: int, url: str) -> None:
|
|
logger.info("[naver_blog] start run=%s url=%s", analysis_run_id, url)
|
|
await set_naver_blog_status(row_id, "processing")
|
|
data = await NaverClient(get_env("NAVER_CLIENT_ID"), get_env("NAVER_CLIENT_SECRET")).get_blog_rss(url)
|
|
await save_naver_blog_raw_data(row_id, data)
|
|
logger.info("[naver_blog] done run=%s", analysis_run_id)
|
|
await check_and_advance_analysis(analysis_run_id)
|
|
|
|
|
|
async def collect_youtube(analysis_run_id: str, row_id: int, url: str) -> None:
|
|
logger.info("[youtube] start run=%s url=%s", analysis_run_id, url)
|
|
await set_youtube_status(row_id, "processing")
|
|
data = await YouTubeClient(get_env("YOUTUBE_API_KEY")).get_channel(url)
|
|
await save_youtube_raw_data(row_id, data)
|
|
logger.info("[youtube] done run=%s", analysis_run_id)
|
|
await check_and_advance_analysis(analysis_run_id)
|
|
|
|
|
|
async def collect_gangnam_unni(analysis_run_id: str, row_id: int, url: str) -> None:
|
|
logger.info("[gangnam_unni] start run=%s url=%s", analysis_run_id, url)
|
|
await set_gangnam_unni_status(row_id, "processing")
|
|
data = await FirecrawlClient(get_env("FIRECRAWL_API_KEY")).get_gangnam_unni(url)
|
|
await save_gangnam_unni_raw_data(row_id, data)
|
|
logger.info("[gangnam_unni] done run=%s", analysis_run_id)
|
|
await check_and_advance_analysis(analysis_run_id)
|
|
|
|
|
|
async def collect_clinic_info(analysis_run_id: str, hospital_id: str, url: str) -> None:
|
|
logger.info("[clinic] start run=%s url=%s", analysis_run_id, url)
|
|
await execute("UPDATE hospital_baseinfo SET status = 'processing' WHERE hospital_id = %s", (hospital_id,))
|
|
data = await FirecrawlClient(get_env("FIRECRAWL_API_KEY")).fetch_clinic_info(url)
|
|
await save_hospital_raw_data(hospital_id, data, analysis_run_id=analysis_run_id)
|
|
logger.info("[clinic] done run=%s", analysis_run_id)
|
|
await check_and_advance_analysis(analysis_run_id)
|