o2o-infinith-demo/supporters/scripts/discover_channels.mjs

211 lines
19 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

// 채널 발견 + 엔티티 검증 (datacrawling 스킬 §1~§2 를 코드로). 병원 이름 또는 URL 하나로 유튜브·인스타·페이스북·네이버 블로그·틱톡·카카오·강남언니·네이버 플레이스를 찾고,
// 후보마다 "이 병원의 것인가"를 근거 점수로 판정한다. 부재는 다섯 갈래를 전부 돈 뒤에만 NOT_FOUND, 아니면 UNVERIFIED.
//
// node scripts/discover_channels.mjs --clinic <id> (--name <상호> | --url <홈>) [--evidence <dir>] [--out channels.json] [--registry ../data/clinic_social_registry.json]
//
// 갈래: ① 사이트 전수(홈·법무·소개·연락처 HTML + JSON-LD sameAs + 근거 수집 링크) ② 웹검색(Firecrawl search·네이버 webkr/blog·YouTube API 채널 검색)
// ③ 핸들 변형(도메인 기반 base × 구분자 × 접미사, 존재 확인만) ④ 레지스트리(data/clinic_social_registry.json) ⑤ 플랫폼 내부 검색(YouTube API)
// 판정: 상호 역링크(+3) · 프로필 제목에 상호(+2) · 프로필 설명에 홈페이지 도메인/전화(+2) · 레지스트리(+2) · 검색 스니펫에 상호+플랫폼(+1)
// ≥3 CONFIRMED · 1~2 CANDIDATE(사람 확인) · 존재 404 HANDLE_DEAD · 접근 차단 BLOCKED · 후보 0 NOT_FOUND(전 갈래 실행 시)/UNVERIFIED
import { readFileSync, writeFileSync, existsSync, readdirSync } from 'node:fs';
import { join, resolve, dirname } from 'node:path';
import { fileURLToPath } from 'node:url';
const here = fileURLToPath(new URL('.', import.meta.url));
const args = process.argv.slice(2);
const opt = (k, d) => { const i = args.indexOf(`--${k}`); return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : d; };
(function loadEnv() { let dir = here; for (let i = 0; i < 4; i++) { const p = join(dir, '.env'); if (existsSync(p)) for (const line of readFileSync(p, 'utf8').split('\n')) { const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)\s*$/); if (m && !process.env[m[1]]) process.env[m[1]] = m[2].replace(/^["']|["']$/g, ''); } dir = dirname(dir); } })();
const KEYS = { yt: process.env.YOUTUBE_API_KEY, fc: process.env.FIRECRAWL_API_KEY, nid: process.env.NAVER_CLIENT_ID, nsec: process.env.NAVER_CLIENT_SECRET };
const clinic = opt('clinic'); let name = opt('name'); let home = opt('url');
if (!clinic || (!name && !home)) { console.error('--clinic 과 --name 또는 --url 필요'); process.exit(2); }
const OUT = resolve(opt('out', `channels.${clinic}.json`));
const EV = opt('evidence') ? resolve(opt('evidence')) : null;
const REG = resolve(opt('registry', join(here, '../../data/clinic_social_registry.json')));
const today = new Date().toISOString().slice(0, 10);
const UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0 Safari/537.36 INFINITH-supporters-discover/1.0 (+mailto:o2oteam@o2o.kr)';
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
const log = (s) => console.log(s);
// ---------- 플랫폼 패턴 (supabase/functions/_shared/extractSocialLinks.ts 이식 + 병의원 표면) ----------
const PLATFORMS = {
youtube: { re: /(?:www\.|m\.)?youtube\.com\/(?:@([a-zA-Z0-9._-]+)|channel\/(UC[a-zA-Z0-9_-]{20,})|c\/([a-zA-Z0-9._-]+)|user\/([a-zA-Z0-9._-]+))/i, key: (m) => (m[2] ? m[2] : '@' + (m[1] || m[3] || m[4])), url: (k) => (k.startsWith('UC') ? `https://www.youtube.com/channel/${k}` : `https://www.youtube.com/${k}`) },
instagram: { re: /(?:www\.)?instagram\.com\/([a-zA-Z0-9._]{2,30})\/?(?:[?#]|$)/i, key: (m) => m[1].toLowerCase(), url: (k) => `https://www.instagram.com/${k}/`, skip: /^(p|reel|reels|explore|accounts|stories|share|about|legal|directory|tv)$/i },
facebook: { re: /(?:www\.|m\.)?facebook\.com\/([a-zA-Z0-9._-]{2,60})\/?(?:[?#]|$)/i, key: (m) => m[1], url: (k) => `https://www.facebook.com/${k}`, skip: /^(sharer|share|login|help|pages|events|groups|marketplace|watch|gaming|privacy|policies|tr|dialog|plugins|photo|video|reel|profile\.php|people|hashtag|search)$/i },
naverBlog: { re: /blog\.naver\.com\/([a-zA-Z0-9_-]{2,40})/i, key: (m) => m[1].toLowerCase(), url: (k) => `https://blog.naver.com/${k}`, skip: /^(PostView|BlogHome|postview)$/i },
tiktok: { re: /(?:www\.)?tiktok\.com\/@([a-zA-Z0-9._-]+)/i, key: (m) => m[1].toLowerCase(), url: (k) => `https://www.tiktok.com/@${k}` },
kakao: { re: /pf\.kakao\.com\/([a-zA-Z0-9_-]+)/i, key: (m) => m[1], url: (k) => `https://pf.kakao.com/${k}` },
gangnamunni: { re: /gangnamunni\.com\/hospitals\/(\d+)/i, key: (m) => m[1], url: (k) => `https://www.gangnamunni.com/hospitals/${k}` },
naverPlace: { re: /(?:m\.)?place\.naver\.com\/(?:hospital|place)\/(\d+)|map\.naver\.com\/(?:p\/entry\/place|v5\/entry\/place)\/(\d+)/i, key: (m) => m[1] || m[2], url: (k) => `https://m.place.naver.com/hospital/${k}` },
babitalk: { re: /babitalk\.com\/hospitals\/(\d+)/i, key: (m) => m[1], url: (k) => `https://babitalk.com/hospitals/${k}` },
};
const cands = Object.fromEntries(Object.keys(PLATFORMS).map((p) => [p, new Map()])); // platform → key → { key, url, sources:Set, hits:[] }
function addCandidate(rawUrl, source, note = '') {
for (const [p, def] of Object.entries(PLATFORMS)) {
const m = String(rawUrl).match(def.re); if (!m) continue;
const k = def.key(m); if (!k || (def.skip && def.skip.test(k))) return;
const c = cands[p].get(k) ?? { platform: p, key: k, url: def.url(k), sources: new Set(), notes: [] };
c.sources.add(source); if (note) c.notes.push(note); cands[p].set(k, c); return;
}
}
const extractAll = (text, source, note) => { for (const m of String(text).matchAll(/https?:\/\/[^\s"'<>)\]]+/g)) addCandidate(m[0], source, note); };
// ---------- 0. 이름 ↔ URL 보완 ----------
async function fetchText(url, tries = 1) {
for (let i = 0; i <= tries; i++) {
try { const r = await fetch(url, { headers: { 'user-agent': UA, accept: 'text/html,*/*' }, redirect: 'follow', signal: AbortSignal.timeout(20000) }); return { status: r.status, url: r.url, body: r.ok ? await r.text() : '' }; }
catch (e) { if (i === tries) return { status: 0, url, body: '', error: e.message }; await sleep(800); }
}
}
async function fcSearch(query, limit = 8) {
if (!KEYS.fc) return [];
try { const r = await fetch('https://api.firecrawl.dev/v1/search', { method: 'POST', headers: { 'content-type': 'application/json', authorization: `Bearer ${KEYS.fc}` }, body: JSON.stringify({ query, limit }), signal: AbortSignal.timeout(30000) }); const j = await r.json(); return (j.data ?? []).map((x) => ({ url: x.url, title: x.title ?? '', snippet: x.description ?? '' })); } catch { return []; }
}
async function naverSearch(kind, query, display = 10) {
if (!KEYS.nid) return [];
try { const r = await fetch(`https://openapi.naver.com/v1/search/${kind}.json?query=${encodeURIComponent(query)}&display=${display}`, { headers: { 'X-Naver-Client-Id': KEYS.nid, 'X-Naver-Client-Secret': KEYS.nsec } }); const j = await r.json(); return (j.items ?? []).map((x) => ({ url: x.link, title: (x.title ?? '').replace(/<[^>]+>/g, ''), snippet: (x.description ?? '').replace(/<[^>]+>/g, '') })); } catch { return []; }
}
const PLATFORM_HOST = /youtube\.com|youtu\.be|instagram\.com|facebook\.com|blog\.naver\.com|tiktok\.com|kakao\.com|gangnamunni\.com|place\.naver\.com|map\.naver\.com|babitalk\.com|naver\.com|google\.|daum\.net|tistory\.com|namu\.wiki|wikipedia/i;
const searchLog = [];
if (!home) {
// 상호로 공식 홈페이지 찾기: 플랫폼이 아닌 도메인 중 제목에 상호가 들어간 첫 결과
const q = `${name} 공식 홈페이지`; searchLog.push(q);
const res = [...(await fcSearch(q, 8)), ...(await naverSearch('webkr', name, 10))];
const core = coreName(name);
const hit = res.find((r) => !PLATFORM_HOST.test(r.url) && (r.title.replace(/\s/g, '').includes(core) || r.snippet.replace(/\s/g, '').includes(core)));
if (!hit) { console.error(`홈페이지를 찾지 못함: ${name}`); }
else { const u = new URL(hit.url); home = `${u.protocol}//${u.host}`; log(`홈페이지 추정: ${home} (${hit.title.slice(0, 40)})`); }
}
const homeHtml = home ? await fetchText(home) : { body: '' };
if (!name) {
const t = homeHtml.body.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1] ?? '';
const og = homeHtml.body.match(/property="og:site_name"\s+content="([^"]+)"/i)?.[1] ?? '';
name = (og || t).replace(/[|–-].*$/, '').replace(/당신만 봅니다/, '').trim() || clinic;
log(`상호 추정: ${name}`);
}
function coreName(n) { return String(n).replace(/\s/g, '').replace(/(성형외과|의원|피부과|클리닉|병원|치과|한의원|WJ|wj)/gi, '') || String(n).replace(/\s/g, ''); }
const core = coreName(name);
const domainBase = home ? new URL(home).host.replace(/^www\./, '').split('.')[0].replace(/^k-|^the-?/, '') : '';
const phone = (homeHtml.body.match(/0\d{1,2}-\d{3,4}-\d{4}/g) ?? [])[0] ?? '';
log(`상호 ${name} · 핵심어 ${core} · 도메인 base ${domainBase} · 전화 ${phone || '?'}`);
// ---------- ① 사이트 전수 ----------
const sitePages = [];
if (home) {
const paths = ['/', '/privacy', '/privacy-policy', '/service/private', '/terms', '/service/terms', '/about', '/hospitalinfo/about', '/contact', '/counsel', '/sitemap.xml'];
for (const p of paths) {
const r = p === '/' ? { ...homeHtml, url: home } : await fetchText(home + p);
if (r.status && r.status < 400 && r.body) {
sitePages.push(p);
extractAll(r.body, 'site', p);
for (const m of r.body.matchAll(/"sameAs"\s*:\s*(\[[^\]]*\]|"[^"]+")/g)) extractAll(m[1], 'site:sameAs', p);
for (const m of r.body.matchAll(/<meta[^>]+(?:property|name)="(?:og:see_also|twitter:site)"[^>]+content="([^"]+)"/g)) extractAll(m[1], 'site:meta', p);
}
await sleep(200);
}
}
if (EV && existsSync(join(EV, 'index.json'))) {
const idx = JSON.parse(readFileSync(join(EV, 'index.json'), 'utf8'));
for (const p of idx.pages) { try { const pg = JSON.parse(readFileSync(join(EV, p.file), 'utf8')); for (const l of pg.links ?? []) addCandidate(String(l.href ?? l.url ?? l ?? ''), 'evidence', p.url); } catch { /* */ } }
}
const homeReciprocal = (homeHtml.body || '').toLowerCase();
// ---------- ④ 레지스트리 ----------
let registry = null;
if (existsSync(REG)) {
const reg = JSON.parse(readFileSync(REG, 'utf8'));
const key = Object.keys(reg).find((k) => k.replace(/\s/g, '') === name.replace(/\s/g, '') || coreName(k) === core);
if (key) { registry = reg[key]; for (const list of Object.values(registry)) for (const u of list ?? []) addCandidate(u, 'registry'); log(`레지스트리: ${key} (${Object.values(registry).flat().length}개 URL)`); }
}
// ---------- ② 웹검색 + ⑤ 플랫폼 검색 ----------
const searchHits = []; // { url, title, snippet, query }
const queries = [
`"${name}" 인스타그램`, `"${name}" 유튜브`, `"${name}" 페이스북`, `"${name}" 블로그`, `${name} 강남언니`, `${name} 네이버 플레이스`,
`site:instagram.com ${core}`, `site:youtube.com ${name}`, `site:facebook.com ${name}`,
];
for (const q of queries) { searchLog.push(q); const res = await fcSearch(q, 8); for (const r of res) { searchHits.push({ ...r, query: q }); addCandidate(r.url, 'search', q); extractAll(r.snippet, 'search:snippet', q); } await sleep(300); }
for (const q of [`${name} 인스타그램`, `${name} 유튜브`, `${name} 페이스북`, `${name} 블로그`]) { searchLog.push('naver:' + q); for (const r of await naverSearch('webkr', q, 10)) { searchHits.push({ ...r, query: 'naver:' + q }); addCandidate(r.url, 'search', q); } }
for (const r of await naverSearch('blog', name, 10)) { searchHits.push({ ...r, query: 'naver:blog' }); addCandidate(r.url, 'search:naverblog', name); }
if (KEYS.yt) {
for (const q of [name, `${name} 성형외과`]) {
try { const r = await fetch(`https://www.googleapis.com/youtube/v3/search?part=snippet&type=channel&q=${encodeURIComponent(q)}&maxResults=5&key=${KEYS.yt}`); const j = await r.json(); for (const it of j.items ?? []) { addCandidate(`https://www.youtube.com/channel/${it.id.channelId}`, 'youtube:search', q); searchHits.push({ url: `https://www.youtube.com/channel/${it.id.channelId}`, title: it.snippet.title, snippet: it.snippet.description, query: 'yt:' + q }); } } catch { /* */ }
}
}
// ---------- ③ 핸들 변형 (존재 확인만) ----------
const variantsTried = [];
if (domainBase && domainBase.length >= 3) {
const SUF = ['', 'official', 'ps', 'clinic', 'plasticsurgery', 'kr', 'korea', 'seoul'];
const SEP = { instagram: ['', '.', '_'], youtube: ['', '_', '-'], tiktok: ['', '.', '_'] };
const vs = new Set(); for (const s of SUF) for (const sep of ['', '.', '_', '-']) vs.add(s ? `${domainBase}${sep}${s}` : domainBase);
for (const p of ['instagram', 'youtube']) {
for (const h of [...vs].filter((h) => SEP[p].some((sep) => !h.includes('.') && !h.includes('_') && !h.includes('-') || h.includes(sep)))) {
if (cands[p].has(p === 'youtube' ? '@' + h : h)) continue;
variantsTried.push(`${p}:${h}`);
if (p === 'youtube' && KEYS.yt) { try { const r = await fetch(`https://www.googleapis.com/youtube/v3/channels?part=id&forHandle=${encodeURIComponent(h)}&key=${KEYS.yt}`); const j = await r.json(); if (j.items?.length) addCandidate(`https://www.youtube.com/channel/${j.items[0].id}`, 'variant', h); } catch { /* */ } }
if (p === 'instagram') { const r = await fetchText(`https://www.instagram.com/${h}/`, 0); if (r.status === 200 && !/Page Not Found|페이지를 찾을 수 없|login/i.test(r.body.slice(0, 3000))) addCandidate(`https://www.instagram.com/${h}/`, 'variant', h); await sleep(400); }
if (variantsTried.length > 40) break;
}
}
}
// ---------- 검증 ----------
const ytChannels = new Map();
async function ytInfo(key) {
if (!KEYS.yt) return null;
const param = key.startsWith('UC') ? `id=${key}` : `forHandle=${encodeURIComponent(key.slice(1))}`;
try { const r = await fetch(`https://www.googleapis.com/youtube/v3/channels?part=snippet,statistics&${param}&key=${KEYS.yt}`); const j = await r.json(); const c = j.items?.[0]; if (!c) return { exists: false }; return { exists: true, id: c.id, title: c.snippet.title, description: c.snippet.description ?? '', handle: c.snippet.customUrl, subscribers: Number(c.statistics.subscriberCount ?? 0), videos: Number(c.statistics.videoCount ?? 0), views: Number(c.statistics.viewCount ?? 0) }; } catch { return null; }
}
async function profileInfo(p, c) {
if (p === 'youtube') return ytInfo(c.key);
if (p === 'naverBlog') { const r = await fetchText(`https://rss.blog.naver.com/${c.key}.xml`, 0); if (r.status === 404) return { exists: false }; if (!r.body) return null; return { exists: true, title: r.body.match(/<title>([^<]*)<\/title>/)?.[1] ?? '', description: r.body.match(/<description>([^<]*)<\/description>/)?.[1] ?? '' }; }
const r = await fetchText(c.url, 0);
if (r.status === 404) return { exists: false };
if (!r.body || r.status !== 200) return null; // 차단·로그인 요구
const og = (prop) => r.body.match(new RegExp(`<meta[^>]+(?:property|name)="${prop}"[^>]+content="([^"]*)"`, 'i'))?.[1] ?? '';
const title = og('og:title') || r.body.match(/<title[^>]*>([^<]*)<\/title>/i)?.[1] || '';
if (/login|로그인|Page Not Found|콘텐츠를 사용할 수 없|isn't available/i.test(title) && !title.includes(core)) return null;
return { exists: true, title, description: og('og:description') || og('description') };
}
const norm = (s) => String(s ?? '').replace(/\s/g, '').toLowerCase();
const domainHost = home ? new URL(home).host.replace(/^www\./, '') : '';
const results = {};
for (const [p, map] of Object.entries(cands)) {
results[p] = [];
for (const c of map.values()) {
const info = await profileInfo(p, c).catch(() => null);
const ev = []; let score = 0;
const hasSite = [...c.sources].some((s) => s.startsWith('site'));
const hostPath = c.url.replace(/^https?:\/\/(www\.|m\.)?/, '').replace(/\/$/, '').toLowerCase();
// 역링크는 경로 경계까지 맞아야 한다 (instagram.com/wonjin 이 instagram.com/wonjin_official 안에 들어 있는 것은 역링크가 아니다)
const reciprocal = hasSite || new RegExp(hostPath.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') + '(?![a-z0-9._-])').test(homeReciprocal);
if (reciprocal) { score += 3; ev.push('홈페이지에서 역링크'); }
if (c.sources.has('registry')) { score += 2; ev.push('레지스트리 등재'); }
if (info?.exists) {
const t = norm(info.title), d = norm(info.description);
if (t.includes(norm(core)) || (domainBase && t.includes(domainBase))) { const w = ['gangnamunni', 'babitalk', 'naverPlace'].includes(p) ? 3 : 2; score += w; ev.push(`프로필 제목에 상호: "${String(info.title).slice(0, 40)}"`); }
if (domainHost && d.includes(domainHost.toLowerCase())) { score += 2; ev.push('프로필 설명에 홈페이지 도메인'); }
else if (phone && d.replace(/[-.]/g, '').includes(phone.replace(/-/g, ''))) { score += 2; ev.push('프로필 설명에 대표 전화'); }
}
const snip = searchHits.filter((h) => norm(h.url).includes(norm(c.key)) && (norm(h.title + h.snippet).includes(norm(core))));
if (snip.length && !ev.some((e) => e.startsWith('프로필 제목'))) { score += 1; ev.push(`검색 결과 제목/요약에 상호 (${snip[0].query})`); }
let status = info === null && !hasSite ? 'BLOCKED' : info && info.exists === false ? 'HANDLE_DEAD' : score >= 3 ? 'CONFIRMED' : score >= 1 ? 'CANDIDATE' : 'UNRELATED';
if (status === 'BLOCKED' && score >= 3) status = 'CONFIRMED';
results[p].push({ platform: p, key: c.key, url: c.url, status, score, evidence: ev, sources: [...c.sources], ...(info?.exists ? { title: info.title, handle: info.handle, channelId: info.id, subscribers: info.subscribers, videos: info.videos, views: info.views } : {}) });
await sleep(150);
}
if (p === 'youtube') { const byId = new Map(); for (const x of results[p]) { const id = x.channelId ?? x.key; const prev = byId.get(id); if (!prev) byId.set(id, x); else { prev.score = Math.max(prev.score, x.score); prev.evidence = [...new Set([...prev.evidence, ...x.evidence])]; prev.sources = [...new Set([...prev.sources, ...x.sources])]; if (x.status === 'CONFIRMED') prev.status = 'CONFIRMED'; } } results[p] = [...byId.values()]; }
results[p].sort((a, b) => b.score - a.score || (b.subscribers ?? 0) - (a.subscribers ?? 0));
}
const allBranches = Boolean(home) && Boolean(KEYS.fc) && Boolean(KEYS.nid) && Boolean(KEYS.yt);
const inventory = Object.entries(results).map(([p, list]) => {
const confirmed = list.filter((x) => x.status === 'CONFIRMED');
const state = confirmed.length ? 'CONFIRMED' : list.some((x) => x.status === 'CANDIDATE') ? 'CANDIDATE' : list.some((x) => x.status === 'BLOCKED') ? 'BLOCKED' : list.length ? 'HANDLE_DEAD' : allBranches ? 'NOT_FOUND' : 'UNVERIFIED';
return { platform: p, state, confirmed: confirmed.length, candidates: list.filter((x) => x.status === 'CANDIDATE').length, tried: `사이트 ${sitePages.length}면 · 검색 ${searchLog.length}건 · 변형 ${variantsTried.filter((v) => v.startsWith(p)).length}` };
});
const out = { clinic, name, url: home ?? null, checkedAt: today, phone: phone || null, branches: { sitePages, searchQueries: searchLog, variantsTried, registry: Boolean(registry), allBranches }, inventory, platforms: results, rules: 'datacrawling 스킬 §1~§2. CONFIRMED=근거 점수 3 이상, CANDIDATE=사람 확인, NOT_FOUND 는 다섯 갈래 전부 실행 시에만' };
writeFileSync(OUT, JSON.stringify(out, null, 1));
for (const row of inventory) log(` ${row.platform.padEnd(11)} ${row.state.padEnd(11)} 확정 ${row.confirmed} 후보 ${row.candidates} ${results[row.platform].filter((x) => x.status === 'CONFIRMED').map((x) => x.key + (x.subscribers ? `(${x.subscribers})` : '')).join(', ')}`);
log(`저장 ${OUT}`);