← back to Ventura Claw
server/connectors/archive.js
130 lines
// Internet Archive / Wayback connector — REAL.
// No API key required for read operations.
// Strict license filter: only items with public-domain / CC0 / CC-BY / CC-BY-SA licenses are returned.
// Docs: https://archive.org/help/wayback_api.php · https://archive.org/advancedsearch.php
// License URL → category. Anything not in this map is rejected from results.
const ALLOWED_LICENSES = {
"creativecommons.org/publicdomain/zero/1.0": "CC0",
"creativecommons.org/publicdomain/mark/1.0": "Public Domain",
"creativecommons.org/licenses/by/4.0": "CC-BY 4.0",
"creativecommons.org/licenses/by/3.0": "CC-BY 3.0",
"creativecommons.org/licenses/by/2.0": "CC-BY 2.0",
"creativecommons.org/licenses/by-sa/4.0": "CC-BY-SA 4.0",
"creativecommons.org/licenses/by-sa/3.0": "CC-BY-SA 3.0",
// CC-NC/CC-ND deliberately excluded — non-commercial restricts SMB use, ND blocks edits.
};
function classifyLicense(url) {
if (!url) return null;
const u = String(url).toLowerCase().replace(/^https?:\/\//, "").replace(/\/$/, "");
for (const k of Object.keys(ALLOWED_LICENSES)) if (u.includes(k)) return ALLOWED_LICENSES[k];
// Public-domain explicit text
if (/\bpublic\s*domain\b/i.test(url)) return "Public Domain";
return null;
}
async function callJson(url, c) {
const r = await fetch(url, {
headers: { "User-Agent": "Commerce-Claw/1.0 (steve@venturaclaw.com)", "Accept": "application/json" },
signal: AbortSignal.timeout(15_000)
});
if (!r.ok) throw new Error(`archive ${r.status}: ${await r.text().then(t => t.slice(0,180)).catch(()=> "")}`);
return r.json();
}
const archive = {
meta: { id: "archive", name: "Archive (Wayback)", category: "data", docsUrl: "https://archive.org/help/wayback_api.php", auth: "none", realImpl: true },
fields: [], // No token needed
configured() { return true; }, // Always available
async health() {
try {
const r = await fetch("https://archive.org/wayback/available?url=example.com", { signal: AbortSignal.timeout(15000) });
return { ok: r.ok, status: r.ok ? "live" : `http ${r.status}`, license_policy: "public-domain + CC0 + CC-BY + CC-BY-SA only" };
} catch (e) { return { ok: false, reason: e.message }; }
},
actions: {
// Wayback Availability — closest snapshot of a URL on a date
async "wayback.available"({ url, timestamp }) {
if (!url) throw new Error("url required");
const qs = new URLSearchParams({ url });
if (timestamp) qs.set("timestamp", timestamp);
const d = await callJson(`https://archive.org/wayback/available?${qs}`);
const snap = d?.archived_snapshots?.closest;
return snap ? { available: true, ...snap } : { available: false };
},
// Search items — license-filtered to public-domain + CC-BY family only.
async "search"({ q, mediatype, rows, page }) {
if (!q) throw new Error("q (query) required");
// Force license filter at query level — much faster than client-side
const license = '(licenseurl:"https://creativecommons.org/publicdomain/zero/1.0/" OR licenseurl:"https://creativecommons.org/publicdomain/mark/1.0/" OR licenseurl:"https://creativecommons.org/licenses/by/4.0/" OR licenseurl:"https://creativecommons.org/licenses/by-sa/4.0/" OR licenseurl:"https://creativecommons.org/licenses/by/3.0/" OR licenseurl:"https://creativecommons.org/licenses/by-sa/3.0/" OR licenseurl:"https://creativecommons.org/licenses/by/2.0/")';
const fullQ = `(${q}) AND ${license}` + (mediatype ? ` AND mediatype:${mediatype}` : "");
const qs = new URLSearchParams({
q: fullQ,
fl: "identifier,title,creator,date,mediatype,licenseurl,description,downloads",
rows: String(Math.min(rows || 25, 200)),
page: String(page || 1),
output: "json"
});
const d = await callJson(`https://archive.org/advancedsearch.php?${qs}`);
const docs = (d.response?.docs || []).map(x => ({
identifier: x.identifier,
title: x.title,
creator: x.creator,
date: x.date,
mediatype: x.mediatype,
license: classifyLicense(x.licenseurl),
license_url: x.licenseurl,
description: typeof x.description === "string" ? x.description.slice(0, 240) : x.description,
downloads: x.downloads,
details_url: `https://archive.org/details/${x.identifier}`,
})).filter(d => d.license); // belt-and-suspenders
return { total: d.response?.numFound || 0, page: d.response?.start || 0, items: docs };
},
// Get full metadata + file list for one item (license enforced)
async "metadata"({ identifier }) {
if (!identifier) throw new Error("identifier required");
const d = await callJson(`https://archive.org/metadata/${encodeURIComponent(identifier)}`);
const license = classifyLicense(d.metadata?.licenseurl);
if (!license) {
return { ok: false, reason: "license not in allowed set", license_url: d.metadata?.licenseurl || null };
}
const files = (d.files || []).filter(f => f.format && /jpeg|png|gif|jpg|mp4|webm|ogg|mp3|pdf|epub|txt/i.test(f.format)).map(f => ({
name: f.name,
format: f.format,
size: f.size ? parseInt(f.size, 10) : null,
url: `https://archive.org/download/${d.metadata.identifier}/${encodeURIComponent(f.name)}`,
}));
return {
ok: true,
identifier: d.metadata.identifier,
title: d.metadata.title,
creator: d.metadata.creator,
date: d.metadata.date,
mediatype: d.metadata.mediatype,
license, license_url: d.metadata.licenseurl,
description: typeof d.metadata.description === "string" ? d.metadata.description : (d.metadata.description?.[0] || ""),
attribution_required: license.startsWith("CC-BY"),
attribution_text: license.startsWith("CC-BY") ? `Source: ${d.metadata.title} by ${d.metadata.creator || "Internet Archive"}, ${license}, archive.org/details/${d.metadata.identifier}` : null,
files,
details_url: `https://archive.org/details/${d.metadata.identifier}`,
};
},
// CDX list of every snapshot of a URL
async "wayback.cdx"({ url, from, to, limit }) {
if (!url) throw new Error("url required");
const qs = new URLSearchParams({ url, output: "json", limit: String(Math.min(limit || 50, 500)) });
if (from) qs.set("from", from);
if (to) qs.set("to", to);
const d = await callJson(`https://web.archive.org/cdx/search/cdx?${qs}`);
if (!Array.isArray(d) || !d.length) return { snapshots: [] };
const [headers, ...rows] = d;
return { snapshots: rows.map(r => Object.fromEntries(headers.map((h, i) => [h, r[i]]))) };
},
}
};
module.exports = archive;