import "@supabase/functions-js/edge-runtime.d.ts"; import { createClient } from "https://esm.sh/@supabase/supabase-js@2"; import type { VerifiedChannels } from "../_shared/verifyHandles.ts"; import { PERPLEXITY_MODEL } from "../_shared/config.ts"; import { captureAllScreenshots, runVisionAnalysis, screenshotErrors, type ScreenshotResult } from "../_shared/visionAnalysis.ts"; import { fetchWithRetry, fetchJsonWithRetry, wrapChannelTask, type ChannelTaskResult } from "../_shared/retry.ts"; const corsHeaders = { "Access-Control-Allow-Origin": "*", "Access-Control-Allow-Headers": "authorization, x-client-info, apikey, content-type", }; const APIFY_BASE = "https://api.apify.com/v2"; interface CollectRequest { reportId: string; clinicId?: string; // V3: clinic UUID runId?: string; // V3: analysis_run UUID } async function runApifyActor(actorId: string, input: Record, token: string): Promise { const res = await fetchWithRetry( `${APIFY_BASE}/acts/${actorId}/runs?token=${token}&waitForFinish=120`, { method: "POST", headers: { "Content-Type": "application/json" }, body: JSON.stringify(input) }, { maxRetries: 1, timeoutMs: 130000, label: `apify:${actorId.split('~')[1] || actorId}` }, ); if (!res.ok) throw new Error(`Apify ${actorId} returned ${res.status}`); const run = await res.json(); const datasetId = run.data?.defaultDatasetId; if (!datasetId) throw new Error(`Apify ${actorId}: no dataset returned`); const itemsRes = await fetchWithRetry( `${APIFY_BASE}/datasets/${datasetId}/items?token=${token}&limit=20`, undefined, { maxRetries: 1, timeoutMs: 30000, label: `apify-dataset:${actorId.split('~')[1] || actorId}` }, ); return itemsRes.json(); } /** * Phase 2: Collect Channel Data * * Uses verified handles from Phase 1 (stored in DB) to collect ALL raw data * from each channel in parallel. Also runs market analysis via Perplexity. */ Deno.serve(async (req) => { if (req.method === "OPTIONS") { return new Response("ok", { headers: corsHeaders }); } try { const { reportId, clinicId: inputClinicId, runId: inputRunId } = (await req.json()) as CollectRequest; if (!reportId) throw new Error("reportId is required"); // Read Phase 1 results from DB const supabaseUrl = Deno.env.get("SUPABASE_URL")!; const supabaseKey = Deno.env.get("SUPABASE_SERVICE_ROLE_KEY")!; const supabase = createClient(supabaseUrl, supabaseKey); const { data: row, error: fetchError } = await supabase .from("marketing_reports") .select("*") .eq("id", reportId) .single(); if (fetchError || !row) throw new Error(`Report not found: ${fetchError?.message}`); const verified = row.verified_channels as VerifiedChannels; const clinicName = row.clinic_name || ""; const address = row.scrape_data?.clinic?.address || ""; const services: string[] = row.scrape_data?.clinic?.services || []; await supabase.from("marketing_reports").update({ status: "collecting" }).eq("id", reportId); const APIFY_TOKEN = Deno.env.get("APIFY_API_TOKEN") || ""; const YOUTUBE_API_KEY = Deno.env.get("YOUTUBE_API_KEY") || ""; const FIRECRAWL_API_KEY = Deno.env.get("FIRECRAWL_API_KEY") || ""; const PERPLEXITY_API_KEY = Deno.env.get("PERPLEXITY_API_KEY") || ""; const NAVER_CLIENT_ID = Deno.env.get("NAVER_CLIENT_ID") || ""; const NAVER_CLIENT_SECRET = Deno.env.get("NAVER_CLIENT_SECRET") || ""; const channelData: Record = {}; const analysisData: Record = {}; const channelTasks: Promise[] = []; // ─── 1. Instagram (multi-account) — try ALL candidates including unverified/unverifiable ─── const igCandidates = (verified.instagram || []).filter((v: Record) => v.handle && v.verified !== false); if (APIFY_TOKEN && igCandidates.length > 0) { channelTasks.push(wrapChannelTask("instagram", async () => { const accounts: Record[] = []; for (const ig of igCandidates) { const items = await runApifyActor("apify~instagram-profile-scraper", { usernames: [ig.handle], resultsLimit: 12 }, APIFY_TOKEN); const profile = (items as Record[])[0]; if (profile && !profile.error) { accounts.push({ username: profile.username, followers: profile.followersCount, following: profile.followsCount, posts: profile.postsCount, bio: profile.biography, isBusinessAccount: profile.isBusinessAccount, externalUrl: profile.externalUrl, igtvVideoCount: profile.igtvVideoCount, latestPosts: ((profile.latestPosts as Record[]) || []).slice(0, 12).map(p => ({ type: p.type, likes: p.likesCount, comments: p.commentsCount, caption: p.caption, timestamp: p.timestamp, })), }); } } if (accounts.length > 0) { channelData.instagramAccounts = accounts; channelData.instagram = accounts[0]; } else { throw new Error("No Instagram profiles found via Apify"); } })); } // ─── 2. YouTube ─── const ytVerified = verified.youtube as Record | null; if (YOUTUBE_API_KEY && (ytVerified?.verified === true || ytVerified?.verified === "unverifiable")) { channelTasks.push(wrapChannelTask("youtube", async () => { const YT = "https://www.googleapis.com/youtube/v3"; let channelId = (ytVerified?.channelId as string) || ""; // If no channelId, try to resolve from handle if (!channelId && ytVerified?.handle) { const h = (ytVerified.handle as string).replace(/^@/, ''); if (h.startsWith('UC')) { channelId = h; } else { for (const param of ['forHandle', 'forUsername']) { const lookupRes = await fetch(`${YT}/channels?part=id&${param}=${h}&key=${YOUTUBE_API_KEY}`); const lookupData = await lookupRes.json(); channelId = lookupData.items?.[0]?.id || ''; if (channelId) break; } } } if (!channelId) throw new Error("Could not resolve YouTube channel ID"); const chRes = await fetchWithRetry(`${YT}/channels?part=snippet,statistics,brandingSettings&id=${channelId}&key=${YOUTUBE_API_KEY}`, undefined, { label: "youtube-api" }); const chData = await chRes.json(); const channel = chData.items?.[0]; if (!channel) throw new Error("YouTube channel not found in API response"); const stats = channel.statistics || {}; const snippet = channel.snippet || {}; // Popular videos const searchRes = await fetchWithRetry(`${YT}/search?part=snippet&channelId=${channelId}&order=viewCount&type=video&maxResults=10&key=${YOUTUBE_API_KEY}`, undefined, { label: "youtube-search" }); const searchData = await searchRes.json(); const videoIds = (searchData.items || []).map((i: Record) => (i.id as Record)?.videoId).filter(Boolean).join(","); let videos: Record[] = []; if (videoIds) { const vRes = await fetchWithRetry(`${YT}/videos?part=snippet,statistics,contentDetails&id=${videoIds}&key=${YOUTUBE_API_KEY}`, undefined, { label: "youtube-videos" }); const vData = await vRes.json(); videos = vData.items || []; } channelData.youtube = { channelId, channelName: snippet.title, handle: snippet.customUrl, description: snippet.description, publishedAt: snippet.publishedAt, thumbnailUrl: snippet.thumbnails?.default?.url, subscribers: parseInt(stats.subscriberCount || "0", 10), totalViews: parseInt(stats.viewCount || "0", 10), totalVideos: parseInt(stats.videoCount || "0", 10), videos: videos.slice(0, 10).map(v => { const vs = v.statistics as Record || {}; const vSnip = v.snippet as Record || {}; const vCon = v.contentDetails as Record || {}; return { title: vSnip.title, views: parseInt(vs.viewCount || "0", 10), likes: parseInt(vs.likeCount || "0", 10), comments: parseInt(vs.commentCount || "0", 10), date: vSnip.publishedAt, duration: vCon.duration, url: `https://www.youtube.com/watch?v=${v.id}`, thumbnail: (vSnip.thumbnails as Record>)?.medium?.url, }; }), }; })); } // ─── 3. Facebook ─── const fbVerified = verified.facebook as Record | null; if (APIFY_TOKEN && (fbVerified?.verified === true || fbVerified?.verified === "unverifiable")) { channelTasks.push(wrapChannelTask("facebook", async () => { const fbUrl = (fbVerified.url as string) || `https://www.facebook.com/${fbVerified.handle}`; const items = await runApifyActor("apify~facebook-pages-scraper", { startUrls: [{ url: fbUrl }] }, APIFY_TOKEN); const page = (items as Record[])[0]; if (page?.title) { channelData.facebook = { pageName: page.title, pageUrl: page.pageUrl || fbUrl, followers: page.followers, likes: page.likes, categories: page.categories, email: page.email, phone: page.phone, website: page.website, address: page.address, intro: page.intro, rating: page.rating, profilePictureUrl: page.profilePictureUrl, }; } else { throw new Error("Facebook page scraper returned no data"); } })); } // ─── 4. 강남언니 ─── const guVerified = verified.gangnamUnni as Record | null; if (FIRECRAWL_API_KEY && guVerified?.verified && guVerified.url) { channelTasks.push(wrapChannelTask("gangnamUnni", async () => { const scrapeRes = await fetchWithRetry("https://api.firecrawl.dev/v1/scrape", { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${FIRECRAWL_API_KEY}` }, body: JSON.stringify({ url: guVerified!.url as string, formats: ["json"], jsonOptions: { prompt: "Extract: hospital name, overall rating (강남언니 rating is always out of 10, NOT out of 5), total review count, doctors with names/ratings/review counts/specialties, procedures offered, address, certifications/badges", schema: { type: "object", properties: { hospitalName: { type: "string" }, rating: { type: "number" }, totalReviews: { type: "number" }, doctors: { type: "array", items: { type: "object", properties: { name: { type: "string" }, rating: { type: "number" }, reviews: { type: "number" }, specialty: { type: "string" } } } }, procedures: { type: "array", items: { type: "string" } }, address: { type: "string" }, badges: { type: "array", items: { type: "string" } }, }, }, }, waitFor: 5000, }), }, { label: "firecrawl-gangnamunni", timeoutMs: 60000 }); if (!scrapeRes.ok) throw new Error(`Firecrawl 강남언니 scrape failed: ${scrapeRes.status}`); const data = await scrapeRes.json(); const hospital = data.data?.json; if (hospital?.hospitalName) { channelData.gangnamUnni = { name: hospital.hospitalName, rawRating: hospital.rating, rating: typeof hospital.rating === 'number' && hospital.rating > 0 && hospital.rating <= 5 ? hospital.rating * 2 : hospital.rating, ratingScale: "/10", totalReviews: hospital.totalReviews, doctors: (hospital.doctors || []).slice(0, 10), procedures: hospital.procedures || [], address: hospital.address, badges: hospital.badges || [], sourceUrl: guVerified!.url as string, }; } else { throw new Error("강남언니 scrape returned no hospital data"); } })); } // ─── 5. Naver Blog + Place ─── if (NAVER_CLIENT_ID && NAVER_CLIENT_SECRET && clinicName) { const naverHeaders = { "X-Naver-Client-Id": NAVER_CLIENT_ID, "X-Naver-Client-Secret": NAVER_CLIENT_SECRET }; channelTasks.push(wrapChannelTask("naverBlog", async () => { // Get verified Naver Blog handle from Phase 1 for official blog URL const nbVerified = verified.naverBlog as Record | null; const officialBlogHandle = nbVerified?.handle ? String(nbVerified.handle) : null; const query = encodeURIComponent(`${clinicName} 후기`); const res = await fetchWithRetry(`https://openapi.naver.com/v1/search/blog.json?query=${query}&display=10&sort=sim`, { headers: naverHeaders }, { label: "naver-blog" }); if (!res.ok) throw new Error(`Naver Blog API returned ${res.status}`); const data = await res.json(); channelData.naverBlog = { totalResults: data.total || 0, searchQuery: `${clinicName} 후기`, // Official blog URL from Phase 1 verified handle officialBlogUrl: officialBlogHandle ? `https://blog.naver.com/${officialBlogHandle}` : null, officialBlogHandle: officialBlogHandle, // Blog mentions (third-party posts, NOT the official blog) posts: (data.items || []).slice(0, 10).map((item: Record) => ({ title: (item.title || "").replace(/<[^>]*>/g, ""), description: (item.description || "").replace(/<[^>]*>/g, ""), link: item.link, bloggerName: item.bloggername, postDate: item.postdate, })), }; })); channelTasks.push(wrapChannelTask("naverPlace", async () => { // Try multiple queries to find the correct place (avoid same-name different clinics) const queries = [ `${clinicName} 성형외과`, `${clinicName} 성형`, clinicName, ]; for (const q of queries) { const query = encodeURIComponent(q); const res = await fetchWithRetry(`https://openapi.naver.com/v1/search/local.json?query=${query}&display=5&sort=comment`, { headers: naverHeaders }, { label: "naver-place" }); if (!res.ok) continue; const data = await res.json(); // Find the best match: prefer category containing 성형 or 피부 const items = (data.items || []) as Record[]; const match = items.find(i => (i.category || '').includes('성형') || (i.category || '').includes('피부') ) || items.find(i => { const name = (i.title || '').replace(/<[^>]*>/g, '').toLowerCase(); return name.includes(clinicName.replace(/성형외과|병원|의원/g, '').trim().toLowerCase()); }) || null; if (match) { channelData.naverPlace = { name: (match.title || "").replace(/<[^>]*>/g, ""), category: match.category, address: match.roadAddress || match.address, telephone: match.telephone, link: match.link, mapx: match.mapx, mapy: match.mapy, }; break; } } })); } // ─── 6. Google Maps ─── if (APIFY_TOKEN && clinicName) { channelTasks.push(wrapChannelTask("googleMaps", async () => { const queries = [`${clinicName} 성형외과`, clinicName, `${clinicName} ${address || "강남"}`]; let items: unknown[] = []; for (const q of queries) { items = await runApifyActor("compass~crawler-google-places", { searchStringsArray: [q], maxCrawledPlacesPerSearch: 3, language: "ko", maxReviews: 10, }, APIFY_TOKEN); if ((items as Record[]).length > 0) break; } const place = (items as Record[])[0]; if (place) { channelData.googleMaps = { name: place.title, rating: place.totalScore, reviewCount: place.reviewsCount, address: place.address, phone: place.phone, clinicWebsite: place.website, // clinic's own website (not Maps URL) mapsUrl: place.url || (place.title ? `https://www.google.com/maps/search/${encodeURIComponent(String(place.title))}` : ''), category: place.categoryName, openingHours: place.openingHours, topReviews: ((place.reviews as Record[]) || []).slice(0, 10).map(r => ({ stars: r.stars, text: r.text, publishedAtDate: r.publishedAtDate, })), }; } else { throw new Error("Google Maps: no matching place found"); } })); } // ─── 7. Market Analysis (Perplexity) ─── if (PERPLEXITY_API_KEY && services.length > 0) { channelTasks.push(wrapChannelTask("marketAnalysis", async () => { const queries = [ { id: "competitors", prompt: `${address || "강남"} 근처 ${services.slice(0, 3).join(", ")} 전문 성형외과/피부과 경쟁 병원 5곳을 분석해줘. 각 병원의 이름, 주요 시술, 온라인 평판, 마케팅 채널을 JSON 형식으로 제공해줘.` }, { id: "keywords", prompt: `한국 ${services.slice(0, 3).join(", ")} 관련 검색 키워드 트렌드. 네이버와 구글에서 월간 검색량이 높은 키워드 20개, 경쟁 강도, 추천 롱테일 키워드를 JSON 형식으로 제공해줘.` }, { id: "market", prompt: `한국 ${services[0] || "성형외과"} 시장 트렌드 2025-2026. 시장 규모, 성장률, 주요 트렌드, 마케팅 채널별 효과를 JSON 형식으로 제공해줘.` }, { id: "targetAudience", prompt: `${clinicName}의 잠재 고객 분석. 연령대별, 성별, 관심 시술, 정보 탐색 채널, 의사결정 요인을 JSON 형식으로 제공해줘.` }, ]; const results = await Promise.allSettled(queries.map(async q => { const res = await fetchWithRetry("https://api.perplexity.ai/chat/completions", { method: "POST", headers: { "Content-Type": "application/json", Authorization: `Bearer ${PERPLEXITY_API_KEY}` }, body: JSON.stringify({ model: PERPLEXITY_MODEL, messages: [ { role: "system", content: "You are a Korean medical marketing analyst. Always respond in Korean. Provide data in valid JSON format." }, { role: "user", content: q.prompt }, ], temperature: 0.3, }), }, { label: `perplexity:${q.id}`, timeoutMs: 60000 }); const data = await res.json(); return { id: q.id, content: data.choices?.[0]?.message?.content || "", citations: data.citations || [] }; })); let successCount = 0; for (const r of results) { if (r.status === "fulfilled") { const { id, content, citations } = r.value; let parsed = content; const jsonMatch = content.match(/```json\n?([\s\S]*?)```/); if (jsonMatch) { try { parsed = JSON.parse(jsonMatch[1]); } catch {} } analysisData[id] = { data: parsed, citations }; successCount++; } } if (successCount === 0) throw new Error("All Perplexity queries failed"); })); } // ─── 8. Vision Analysis: Screenshots + Gemini Vision ─── const GEMINI_API_KEY = Deno.env.get("GEMINI_API_KEY") || ""; let screenshots: ScreenshotResult[] = []; if (FIRECRAWL_API_KEY) { const mainUrl = row.url || ""; const siteMap: string[] = row.scrape_data?.siteMap || []; channelTasks.push(wrapChannelTask("vision", async () => { // Capture screenshots of relevant pages + social channel landings screenshots = await captureAllScreenshots(mainUrl, siteMap, verified, FIRECRAWL_API_KEY); // Run Gemini Vision on captured screenshots if (GEMINI_API_KEY && screenshots.length > 0) { const vision = await runVisionAnalysis(screenshots, GEMINI_API_KEY); channelData.visionAnalysis = vision.merged; channelData.visionPerPage = vision.perPage; } // Store screenshots metadata (NOT base64 — use the GCS URL from Firecrawl) channelData.screenshots = screenshots.map(ss => ({ id: ss.id, url: ss.url, // GCS signed URL (valid ~7 days) channel: ss.channel, capturedAt: ss.capturedAt, caption: ss.caption, sourceUrl: ss.sourceUrl, })); if (screenshots.length === 0) { const debugInfo = screenshotErrors.length > 0 ? screenshotErrors.join(" | ") : "No errors recorded — check FIRECRAWL_API_KEY"; throw new Error(`No screenshots captured: ${debugInfo}`); } })); } // ─── Execute all channel tasks ─── const taskResults = await Promise.all(channelTasks); // ─── Build channelErrors from task results ─── const channelErrors: Record = {}; let failedCount = 0; let successCount = 0; for (const result of taskResults) { if (result.success) { successCount++; } else { failedCount++; channelErrors[result.channel] = { error: result.error || "Unknown error", durationMs: result.durationMs, }; } } const totalTasks = taskResults.length; const isPartial = failedCount > 0 && successCount > 0; const isFullFailure = failedCount > 0 && successCount === 0; const collectionStatus = isFullFailure ? "collection_failed" : isPartial ? "partial" : "collected"; console.log(`[collect] ${successCount}/${totalTasks} tasks succeeded. Status: ${collectionStatus}`); if (failedCount > 0) { console.warn(`[collect] Failed channels:`, JSON.stringify(channelErrors)); } // ─── UNCONDITIONAL Legacy Save: always persist whatever we have ─── await supabase.from("marketing_reports").update({ channel_data: channelData, analysis_data: { clinicName, services, address, analysis: analysisData, analyzedAt: new Date().toISOString() }, channel_errors: channelErrors, status: collectionStatus, updated_at: new Date().toISOString(), }).eq("id", reportId); // ─── V3: channel_snapshots + screenshots + analysis_runs ─── const clinicId = inputClinicId || null; const runId = inputRunId || null; if (clinicId && runId) { try { // Channel snapshots — INSERT one row per channel (time-series!) const snapshotInserts: Record[] = []; const igData = channelData.instagram as Record | undefined; if (igData) { snapshotInserts.push({ clinic_id: clinicId, run_id: runId, channel: 'instagram', handle: igData.username, followers: igData.followers, posts: igData.posts, details: igData, }); } const ytData = channelData.youtube as Record | undefined; if (ytData) { snapshotInserts.push({ clinic_id: clinicId, run_id: runId, channel: 'youtube', handle: ytData.handle || ytData.channelName, followers: ytData.subscribers, posts: ytData.totalVideos, total_views: ytData.totalViews, details: ytData, }); } const fbData = channelData.facebook as Record | undefined; if (fbData) { snapshotInserts.push({ clinic_id: clinicId, run_id: runId, channel: 'facebook', handle: fbData.pageName, followers: fbData.followers, details: fbData, }); } const guData = channelData.gangnamUnni as Record | undefined; if (guData) { snapshotInserts.push({ clinic_id: clinicId, run_id: runId, channel: 'gangnamUnni', handle: guData.name, rating: guData.rating, rating_scale: 10, reviews: guData.totalReviews, details: guData, }); } const gmData = channelData.googleMaps as Record | undefined; if (gmData) { snapshotInserts.push({ clinic_id: clinicId, run_id: runId, channel: 'googleMaps', handle: gmData.name, rating: gmData.rating, rating_scale: 5, reviews: gmData.reviewCount, details: gmData, }); } const nbData = channelData.naverBlog as Record | undefined; if (nbData) { snapshotInserts.push({ clinic_id: clinicId, run_id: runId, channel: 'naverBlog', handle: nbData.officialBlogHandle, details: nbData, }); } if (snapshotInserts.length > 0) { await supabase.from("channel_snapshots").insert(snapshotInserts); } // Screenshots — INSERT evidence rows const screenshotList = (channelData.screenshots || []) as Record[]; if (screenshotList.length > 0) { await supabase.from("screenshots").insert( screenshotList.map(ss => ({ clinic_id: clinicId, run_id: runId, channel: ss.channel, page_type: (ss.id as string || '').split('-')[1] || 'main', url: ss.url, source_url: ss.sourceUrl, caption: ss.caption, })) ); } // Update analysis_run with status + errors await supabase.from("analysis_runs").update({ raw_channel_data: channelData, analysis_data: { clinicName, services, address, analysis: analysisData }, vision_analysis: channelData.visionAnalysis || {}, channel_errors: channelErrors, status: collectionStatus, }).eq("id", runId); } catch (e) { console.error("V3 dual-write error:", e); } } return new Response( JSON.stringify({ success: !isFullFailure, status: collectionStatus, channelData, analysisData, channelErrors: Object.keys(channelErrors).length > 0 ? channelErrors : undefined, partialFailure: isPartial, taskSummary: { total: totalTasks, succeeded: successCount, failed: failedCount }, collectedAt: new Date().toISOString(), }), { headers: { ...corsHeaders, "Content-Type": "application/json" } }, ); } catch (error) { return new Response( JSON.stringify({ success: false, error: error.message }), { status: 500, headers: { ...corsHeaders, "Content-Type": "application/json" } }, ); } });