/** * Retry utility for external API calls. * * Features: * - Exponential backoff with jitter * - Respects Retry-After header (429) * - Permanent failure detection (400/401/403/404 → no retry) * - Per-request timeout via AbortController * - Domain-level rate limiting (e.g., Firecrawl 500ms gap) */ // ─── Types ─── export interface RetryOptions { /** Max number of retries (default: 2) */ maxRetries?: number; /** Backoff delays in ms per attempt (default: [1000, 3000]) */ backoffMs?: number[]; /** HTTP status codes to retry on (default: [429, 500, 502, 503]) */ retryOn?: number[]; /** Per-request timeout in ms (default: 45000) */ timeoutMs?: number; /** Label for logging */ label?: string; } interface RetryResult { response: Response; attempts: number; retried: boolean; } // ─── Domain Rate Limiter ─── const domainLastCall = new Map(); const DOMAIN_INTERVALS: Record = { "api.firecrawl.dev": 500, "api.perplexity.ai": 200, }; function getDomain(url: string): string { try { return new URL(url).hostname; } catch { return ""; } } async function waitForDomainSlot(url: string): Promise { const domain = getDomain(url); const interval = DOMAIN_INTERVALS[domain]; if (!interval) return; const last = domainLastCall.get(domain) || 0; const elapsed = Date.now() - last; if (elapsed < interval) { await new Promise((r) => setTimeout(r, interval - elapsed)); } domainLastCall.set(domain, Date.now()); } // ─── Permanent failure codes (never retry) ─── const PERMANENT_FAILURES = new Set([400, 401, 403, 404, 405, 409, 422]); // ─── Main Function ─── /** * fetch() with automatic retry, timeout, and rate limiting. * * @example * const res = await fetchWithRetry("https://api.example.com/data", { * method: "POST", * headers: { "Content-Type": "application/json" }, * body: JSON.stringify({ query: "test" }), * }, { maxRetries: 2, timeoutMs: 30000, label: "example-api" }); */ export async function fetchWithRetry( url: string, init?: RequestInit, opts?: RetryOptions, ): Promise { const { maxRetries = 2, backoffMs = [1000, 3000], retryOn = [429, 500, 502, 503], timeoutMs = 45000, label = getDomain(url), } = opts || {}; let lastError: Error | null = null; const retrySet = new Set(retryOn); for (let attempt = 0; attempt <= maxRetries; attempt++) { // Rate limit between domain calls await waitForDomainSlot(url); // AbortController for per-request timeout const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { const response = await fetch(url, { ...init, signal: controller.signal, }); clearTimeout(timer); // Success if (response.ok) return response; // Permanent failure — don't retry if (PERMANENT_FAILURES.has(response.status)) { console.warn( `[retry:${label}] Permanent failure ${response.status} on attempt ${attempt + 1}`, ); return response; } // Retryable failure if (retrySet.has(response.status) && attempt < maxRetries) { let delay = backoffMs[attempt] || backoffMs[backoffMs.length - 1] || 3000; // Respect Retry-After header for 429 if (response.status === 429) { const retryAfter = response.headers.get("Retry-After"); if (retryAfter) { const parsed = parseInt(retryAfter, 10); if (!isNaN(parsed)) { delay = Math.max(delay, parsed * 1000); } } } // Add jitter (±20%) delay = delay * (0.8 + Math.random() * 0.4); console.warn( `[retry:${label}] Status ${response.status}, retrying in ${Math.round(delay)}ms (attempt ${attempt + 1}/${maxRetries + 1})`, ); await new Promise((r) => setTimeout(r, delay)); continue; } // Non-retryable or exhausted retries return response; } catch (err) { clearTimeout(timer); lastError = err instanceof Error ? err : new Error(String(err)); if (attempt < maxRetries) { const delay = (backoffMs[attempt] || 3000) * (0.8 + Math.random() * 0.4); console.warn( `[retry:${label}] Network error: ${lastError.message}, retrying in ${Math.round(delay)}ms (attempt ${attempt + 1}/${maxRetries + 1})`, ); await new Promise((r) => setTimeout(r, delay)); continue; } } } throw lastError || new Error(`[retry:${label}] All ${maxRetries + 1} attempts failed`); } // ─── Convenience: JSON fetch with retry ─── export async function fetchJsonWithRetry( url: string, init?: RequestInit, opts?: RetryOptions, ): Promise<{ data: T | null; status: number; error?: string }> { try { const res = await fetchWithRetry(url, init, opts); if (!res.ok) { const text = await res.text().catch(() => ""); return { data: null, status: res.status, error: `HTTP ${res.status}: ${text.slice(0, 200)}` }; } const data = (await res.json()) as T; return { data, status: res.status }; } catch (err) { const msg = err instanceof Error ? err.message : String(err); return { data: null, status: 0, error: msg }; } } // ─── Channel Task Wrapper ─── export interface ChannelTaskResult { channel: string; success: boolean; error?: string; httpStatus?: number; durationMs: number; } /** * Wraps a channel collection task with timing and error capture. * Used by collect-channel-data to track per-channel success/failure. * * @example * const [result, taskMeta] = await wrapChannelTask("instagram", async () => { * // ... collect instagram data ... * channelData.instagram = data; * }); */ export async function wrapChannelTask( channel: string, task: () => Promise, ): Promise { const start = Date.now(); try { await task(); return { channel, success: true, durationMs: Date.now() - start }; } catch (err) { const msg = err instanceof Error ? err.message : String(err); console.error(`[channel:${channel}] Error: ${msg}`); return { channel, success: false, error: msg, durationMs: Date.now() - start, }; } }