`src/lib/websearch.ts`

140 lines
/**
 * Multi-engine web search with graceful fallback.
 * Tries DDG HTML → DDG Lite → Brave Search HTML and returns the first engine that yields results.
 * All three are free, keyless, and public-facing; we rotate when one rate-limits.
 */

const UA =
  "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";

const COMMON_HEADERS = {
  "user-agent": UA,
  accept: "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
  "accept-language": "en-US,en;q=0.9",
};

export type Snippet = { title: string; url: string; body: string };
export type SearchResult = {
  engine: "ddg" | "ddg_lite" | "brave" | null;
  snippets: Snippet[];
  error?: string;
};

const strip = (s: string) =>
  s
    .replace(/<[^>]+>/g, "")
    .replace(/&amp;/g, "&")
    .replace(/&quot;/g, '"')
    .replace(/&#x27;/g, "'")
    .replace(/&nbsp;/g, " ")
    .replace(/\s+/g, " ")
    .trim();

async function tryDdg(q: string): Promise<Snippet[]> {
  const form = new URLSearchParams();
  form.set("q", q);
  const r = await fetch("https://html.duckduckgo.com/html/", {
    method: "POST",
    headers: {
      ...COMMON_HEADERS,
      "content-type": "application/x-www-form-urlencoded",
      referer: "https://html.duckduckgo.com/",
    },
    body: form.toString(),
    signal: AbortSignal.timeout(15000),
  });
  const body = await r.text();
  if (r.status === 202 || /\banomaly\b/i.test(body)) throw new Error("ddg rate-limited");
  if (!r.ok) throw new Error(`ddg HTTP ${r.status}`);
  const titles = [...body.matchAll(/<a[^>]*class="result__a"[^>]*>([\s\S]*?)<\/a>/g)].map((m) => strip(m[1]));
  const urls = [...body.matchAll(/class="result__url"[^>]*>([\s\S]*?)<\/a>/g)].map((m) => strip(m[1]));
  const snips = [...body.matchAll(/class="result__snippet"[^>]*>([\s\S]*?)<\/a>/g)].map((m) => strip(m[1]));
  const out: Snippet[] = [];
  for (let i = 0; i < titles.length && i < 10; i++) {
    if (!titles[i]) continue;
    out.push({ title: titles[i], url: urls[i] ?? "", body: snips[i] ?? "" });
  }
  if (!out.length) throw new Error("ddg returned zero parseable results");
  return out;
}

async function tryDdgLite(q: string): Promise<Snippet[]> {
  const r = await fetch(`https://lite.duckduckgo.com/lite/?q=${encodeURIComponent(q)}`, {
    headers: COMMON_HEADERS,
    signal: AbortSignal.timeout(15000),
  });
  const body = await r.text();
  if (r.status === 202 || /\banomaly\b/i.test(body)) throw new Error("ddg_lite rate-limited");
  if (!r.ok) throw new Error(`ddg_lite HTTP ${r.status}`);
  const rowRe = /<tr[^>]*>[\s\S]*?<\/tr>/g;
  const rows = body.match(rowRe) ?? [];
  const out: Snippet[] = [];
  // DDG Lite alternates: row with result-link (title), row with result-snippet (body), row with link-text (url).
  let cur: Partial<Snippet> = {};
  for (const row of rows) {
    const titleMatch = /<a[^>]*class=['"]result-link['"][^>]*>([\s\S]*?)<\/a>/.exec(row);
    const snipMatch = /class=['"]result-snippet['"][^>]*>([\s\S]*?)<\/td>/.exec(row);
    const urlMatch = /class=['"]link-text['"][^>]*>([\s\S]*?)<\/span>/.exec(row);
    if (titleMatch) {
      if (cur.title) out.push({ title: cur.title!, url: cur.url ?? "", body: cur.body ?? "" });
      cur = { title: strip(titleMatch[1]) };
    } else if (snipMatch && cur.title !== undefined) {
      cur.body = strip(snipMatch[1]);
    } else if (urlMatch && cur.title !== undefined) {
      cur.url = strip(urlMatch[1]);
    }
    if (out.length >= 10) break;
  }
  if (cur.title) out.push({ title: cur.title, url: cur.url ?? "", body: cur.body ?? "" });
  if (!out.length) throw new Error("ddg_lite returned zero parseable results");
  return out;
}

async function tryBrave(q: string): Promise<Snippet[]> {
  const r = await fetch(`https://search.brave.com/search?q=${encodeURIComponent(q)}`, {
    headers: COMMON_HEADERS,
    signal: AbortSignal.timeout(15000),
  });
  const body = await r.text();
  if (r.status === 429) throw new Error("brave rate-limited");
  if (!r.ok) throw new Error(`brave HTTP ${r.status}`);
  // Brave renders server-side via Svelte. Each result has:
  //   title="TITLE TEXT" inside a snippet-title element, and a following snippet-description.
  const titleMatches = [...body.matchAll(/class="[^"]*snippet-title[^"]*"[^>]*title="([^"]+)"/g)].map((m) => strip(m[1]));
  const descMatches = [...body.matchAll(/class="[^"]*snippet-description[^"]*"[^>]*>([\s\S]{0,500}?)<\/div>/g)].map((m) => strip(m[1]));
  const urlMatches = [...body.matchAll(/<cite[^>]*class="[^"]*snippet-url[^"]*"[^>]*>([\s\S]{0,200}?)<\/cite>/g)].map((m) => strip(m[1]));
  const out: Snippet[] = [];
  for (let i = 0; i < titleMatches.length && i < 10; i++) {
    out.push({ title: titleMatches[i], url: urlMatches[i] ?? "", body: descMatches[i] ?? "" });
  }
  if (!out.length) throw new Error("brave returned zero parseable results");
  return out;
}

/**
 * Rotates starting engine across calls so we don't always hammer DDG first.
 * Persists a tiny round-robin counter in-process.
 */
let rrCounter = 0;

export async function webSearch(query: string): Promise<SearchResult> {
  const engines: { name: SearchResult["engine"]; run: () => Promise<Snippet[]> }[] = [
    { name: "ddg", run: () => tryDdg(query) },
    { name: "ddg_lite", run: () => tryDdgLite(query) },
    { name: "brave", run: () => tryBrave(query) },
  ];
  // Round-robin start.
  const start = rrCounter++ % engines.length;
  const ordered = [...engines.slice(start), ...engines.slice(0, start)];
  const errors: string[] = [];
  for (const e of ordered) {
    try {
      const snippets = await e.run();
      return { engine: e.name, snippets };
    } catch (err) {
      errors.push(`${e.name}: ${(err as Error).message}`);
    }
  }
  return { engine: null, snippets: [], error: errors.join(" | ") };
}