add autoproxy for firecrwal

.gitignore

@@ -45,4 +45,6 @@ alembic/versions/*.pyc
 
 test_results/
 
-app/test*
+app/test*
+
+docker-compose.yml

app/integrations/firecrawl.py

@@ -16,7 +16,7 @@ class FirecrawlClient:
             HTTPMethod.POST,
             url=f"{FIRECRAWL_BASE}/scrape",
             headers=self._headers(),
-            json_body={"url": url, "formats": ["json", "links"], "jsonOptions": json_options, "waitFor": wait_for},
+            json_body={"url": url, "formats": ["json", "links"], "jsonOptions": json_options, "waitFor": wait_for, "maxAge": 0},
             label="firecrawl-scrape",
         )
         if not resp or not resp.is_success:
@@ -76,7 +76,7 @@ class FirecrawlClient:
                 "url": url,
                 "formats": ["json", "links", "rawHtml"],
                 "jsonOptions": {
-                    "prompt": "Extract: clinic name (Korean), clinic name (English), address, phone with dash format, business hours, slogan, services offered, doctors with name/title/specialty, brand identity (primary/accent/background/text colors in hex, heading/body fonts, logo URL from the actual header/main <img> src, og:image from <meta property='og:image'> content, favicon URL)",
+                    "prompt": "Extract: 클리닉 이름 - clinicName (Korean), clinic name (English), address, phone with dash format, business hours, slogan, services offered, doctors with name/title/specialty, brand identity (primary/accent/background/text colors in hex, heading/body fonts, logo URL from the actual header/main <img> src, og:image from <meta property='og:image'> content, favicon URL)",
                     "schema": {
                         "type": "object",
                         "properties": {
@@ -127,8 +127,11 @@ class FirecrawlClient:
                     },
                 },
                 "waitFor": 5000,
+                "maxAge": 0,
+                "proxy": "auto",     # 기본 엔진이 차단되는 사이트(예: viewclinic.com)는 자동으로 stealth 프록시로 재시도.
+                "timeout": 120000,   # proxy:auto면 60s로도 충분했지만(실측), 혹시 모르니 여유있게 120s.
             },
-            timeout=60,
+            timeout=150,  # 위 Firecrawl 잡 타임아웃보다 길어야 우리 쪽 HTTP 클라이언트가 먼저 끊지 않음.
             label="firecrawl-clinic-info",
         )
         if not resp or not resp.is_success:
@@ -187,6 +190,7 @@ class FirecrawlClient:
                     },
                 },
                 "waitFor": 5000,
+                "maxAge": 0,
             },
             timeout=60,
             label="firecrawl-gangnamunni",

app/integrations/llm/prompt.py

@@ -13,7 +13,7 @@ from integrations.llm.schemas.report import (
     ScoresInput, ScoresOutput,
     OtherChannelsInput, OtherChannelsOutput
 )
-from integrations.llm.schemas.plan import PlanInput, PlanOutput
+from integrations.llm.schemas.plan import PlanInput, PlanOutput, SummarizeInput, SummarizeOutput
 from integrations.llm.schemas.market import (
     MarketCompetitorsInput, MarketCompetitorsOutput,
     MarketKeywordsInput, MarketKeywordsOutput,
@@ -64,6 +64,13 @@ plan_prompt = Prompt(
     output_class=PlanOutput,
 )
 
+summarize_prompt = Prompt(
+    file_name="summarize_prompt.txt",
+    prompt_model="PLAN_MODEL",
+    input_class=SummarizeInput,
+    output_class=SummarizeOutput,
+)
+
 market_competitors_prompt = Prompt(
     file_name="market_competitors_prompt.txt",
     prompt_model="MARKET_MODEL",

app/integrations/llm/schemas/plan.py

@@ -2,6 +2,15 @@ from typing import Literal
 from pydantic import BaseModel
 
 
+class SummarizeInput(BaseModel):
+    label: str
+    data: str
+
+
+class SummarizeOutput(BaseModel):
+    summary: str
+
+
 class PlanInput(BaseModel):
     clinic_name: str | None = None
     clinic_name_en: str | None = None

app/integrations/llm/temp-prompt/summarize_prompt.txt

@@ -0,0 +1,7 @@
+다음은 "{label}" 원본 데이터입니다.
+
+{data}
+
+위 데이터를 마케팅 플랜 생성에 필요한 핵심 정보만 남기고 간결하게 요약하세요.
+구체적인 수치·날짜·고유명사(채널명, 게시물 제목 등)는 그대로 보존하고, 중복되거나 플랜 작성에 불필요한 메타데이터는 제거하세요.
+요약문 하나의 문자열로만 출력하세요.

app/services/analysis.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import logging
 from urllib.parse import urlparse
@@ -7,7 +8,7 @@ from common.db.run import update_run_report, update_run_plan, select_run_report_
 from common.db.source import select_run_raw_data, select_mainpage_logo_url
 from common.db.market import select_market
 from integrations.llm.llm_service import LLMService
-from integrations.llm.prompt import report_prompt, plan_prompt, youtube_diagnosis_prompt, brand_consistency_prompt, critical_issues_prompt, transformation_prompt, roadmap_prompt, scores_prompt, other_channels_prompt
+from integrations.llm.prompt import report_prompt, plan_prompt, summarize_prompt, youtube_diagnosis_prompt, brand_consistency_prompt, critical_issues_prompt, transformation_prompt, roadmap_prompt, scores_prompt, other_channels_prompt
 from integrations.llm.schemas.report import ReportOutput, ClinicSnapshot, YouTubeAudit, BrandConsistencyOutput, CriticalIssuesOutput, DiagnosisItem, TransformationProposal, RoadmapOutput, RoadmapMonth, ScoresOutput, ChannelScore, WebsiteAudit, OtherChannelsOutput, OtherChannel
 from services.branding import analyze_branding
 from services.instagram_audit import build_instagram_audit
@@ -49,32 +50,52 @@ async def generate_plan(analysis_run_id: str) -> PlanOutput:
     def _json(v) -> str | None:
         return json.dumps(v, ensure_ascii=False) if v else None
 
-    input_data = {
-        "clinic_name":            clinic.get("clinicName"),
-        "clinic_name_en":         clinic.get("clinicNameEn"),
-        "address":                clinic.get("address"),
-        "phone":                  clinic.get("phone"),
-        "slogan":                 clinic.get("slogan"),
-        "services":               json.dumps(clinic.get("services", []), ensure_ascii=False),
-        "doctors":                json.dumps(clinic.get("doctors", []), ensure_ascii=False),
+    # map: 큰 입력은 LLM으로 압축 요약해 100KB 초과 에러를 방지하고, 작은 입력은 그대로 둔다.
+    large_fields = {
         "report":                 _json(report),
         "market_competitors":     _json(market.get("competitors")),
         "market_keywords":        _json(market.get("keywords")),
         "market_trend":           _json(market.get("trend")),
         "market_target_audience": _json(market.get("target_audience")),
         "tiktok":                 _json(tiktok),
-        "instagram":           _json(instagram),
-        "facebook":            _json(facebook),
-        "naver_blog":             _json(_naver_blog_summary(naver_blog)),
+        "instagram":              _json(instagram),
+        "facebook":               _json(facebook),
         "naver_cafe":             _json(naver_cafe),
-        "kakao_talk":             _json(kakaotalk),
         "channel_logos":          _json(branding.get("channelLogos")),
         "brand_assets":           _json(branding.get("brandAssets")),
     }
+    summarized = dict(zip(
+        large_fields.keys(),
+        await asyncio.gather(*(_summarize(label, data) for label, data in large_fields.items())),
+    ))
+
+    # reduce: 요약된 입력을 모아 최종 플랜 생성.
+    input_data = {
+        "clinic_name":    clinic.get("clinicName"),
+        "clinic_name_en": clinic.get("clinicNameEn"),
+        "address":        clinic.get("address"),
+        "phone":          clinic.get("phone"),
+        "slogan":         clinic.get("slogan"),
+        "services":       json.dumps(clinic.get("services", []), ensure_ascii=False),
+        "doctors":        json.dumps(clinic.get("doctors", []), ensure_ascii=False),
+        "naver_blog":     _json(_naver_blog_summary(naver_blog)),
+        "kakao_talk":     _json(kakaotalk),
+        **summarized,
+    }
 
     return await LLMService(provider="perplexity").generate(plan_prompt, input_data)
 
 
+_SUMMARIZE_THRESHOLD = 4000  # 이 길이(문자 수)를 넘는 입력만 요약 LLM 호출 (불필요한 호출 방지)
+
+
+async def _summarize(label: str, data: str | None) -> str | None:
+    if not data or len(data) <= _SUMMARIZE_THRESHOLD:
+        return data
+    result = await LLMService(provider="perplexity").generate(summarize_prompt, {"label": label, "data": data})
+    return result.summary
+
+
 def _build_clinic_snapshot(mainpage: dict, gangnam_unni: dict, brand_assets: dict, logo_url: str | None) -> dict:
     snapshot: dict = {}
     doctors = gangnam_unni.get("doctors", [])