← back to Norma
agents/datasource-agent/skills/scrape-petitions.js
763 lines
/**
* Scrape Petitions Skill
*
* Scrapes petition platforms for trending petitions, legislative bills,
* and civic engagement campaigns. Uses Gemini 2.0 Flash to score relevance
* to nonprofit advocacy and civic engagement.
* Results are stored in the trending_petitions table.
*/
const fetch = require('node-fetch');
const cheerio = require('cheerio');
const { logAction } = require('../../shared/audit-logger');
const { PETITION_PLATFORMS } = require('../lib/petition-registry');
const AGENT = 'datasource-agent';
const PLATFORM = 'petitions';
const GEMINI_URL =
'https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key=${GOOGLE_API_KEY}';
const DB_URL = process.env.DATABASE_URL || 'postgresql://dw_admin@127.0.0.1:5432/sdcc';
let _pool;
function getPool() {
if (!_pool) {
const { Pool } = require('pg');
_pool = new Pool({ connectionString: DB_URL, max: 5 });
}
return _pool;
}
// ──────────────────────────────────────
// HTTP helpers
// ──────────────────────────────────────
/**
* Fetch a URL with a 15-second timeout.
*
* @param {string} url - URL to fetch
* @param {Object} [options={}] - Additional fetch options
* @returns {Promise<Response>}
*/
async function fetchWithTimeout(url, options = {}) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), 15000);
try {
const res = await fetch(url, {
signal: controller.signal,
headers: {
'User-Agent': 'Norma-DataSource-Agent/1.0 (research; non-commercial)',
'Accept': 'application/json, text/html, */*',
...options.headers,
},
...options,
});
clearTimeout(timeout);
return res;
} catch (err) {
clearTimeout(timeout);
throw err;
}
}
// ──────────────────────────────────────
// Platform-specific scrapers
// ──────────────────────────────────────
/**
* Scrape UK Parliament Petitions (JSON API).
*
* @returns {Promise<Array<Object>>} Normalized petition objects
*/
async function scrapeUKParliament() {
const items = [];
try {
const url = 'https://petition.parliament.uk/petitions.json?state=open&page=1';
const res = await fetchWithTimeout(url);
if (!res.ok) return items;
const json = await res.json();
const petitions = json.data || [];
for (const p of petitions.slice(0, 20)) {
const attrs = p.attributes || {};
const id = p.id || String(attrs.action || '').substring(0, 40);
items.push({
platform: 'uk_parliament',
petition_id: String(id),
title: (attrs.action || '').substring(0, 500),
description: (attrs.background || '').substring(0, 2000),
url: `https://petition.parliament.uk/petitions/${id}`,
signature_count: attrs.signature_count || 0,
signature_goal: attrs.response_threshold_reached_at ? 100000 : 10000,
category: attrs.topics ? attrs.topics.join(', ') : 'general',
creator: attrs.creator_name || 'Unknown',
target: attrs.state || 'open',
created_date: attrs.created_at ? attrs.created_at.substring(0, 10) : null,
});
}
console.log(`[${AGENT}] UK Parliament: found ${items.length} petitions`);
} catch (err) {
console.error(`[${AGENT}] UK Parliament scrape error:`, err.message);
}
return items;
}
/**
* Scrape OpenPetition EU (HTML scrape via cheerio).
*
* @returns {Promise<Array<Object>>} Normalized petition objects
*/
async function scrapeOpenPetitionEU() {
const items = [];
try {
const res = await fetchWithTimeout('https://www.openpetition.eu/petition/neu', {
headers: {
'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'en-US,en;q=0.9',
},
});
if (!res.ok) return items;
const html = await res.text();
const $ = cheerio.load(html);
// OpenPetition lists petitions in .petition-teaser or similar containers
// The site uses various CSS class patterns; try common selectors
const selectors = [
'.petition-teaser',
'.petition-item',
'.petition-list-item',
'article.petition',
'.teaser',
'.list-item',
];
let petitionEls = $([]);
for (const sel of selectors) {
petitionEls = $(sel);
if (petitionEls.length > 0) break;
}
// Fallback: look for links to /petition/online/ pattern
if (petitionEls.length === 0) {
$('a[href*="/petition/online/"]').each((i, el) => {
if (i >= 20) return false;
const $el = $(el);
const href = $el.attr('href') || '';
const title = $el.text().trim();
if (title && href) {
const fullUrl = href.startsWith('http') ? href : `https://www.openpetition.eu${href}`;
const petId = href.split('/').filter(Boolean).pop() || `eu-${i}`;
items.push({
platform: 'openpetition_eu',
petition_id: petId,
title: title.substring(0, 500),
description: '',
url: fullUrl,
signature_count: 0,
signature_goal: null,
category: 'general',
creator: 'Unknown',
target: 'open',
created_date: null,
});
}
});
} else {
petitionEls.each((i, el) => {
if (i >= 20) return false;
const $el = $(el);
const titleEl = $el.find('h2, h3, .title, a').first();
const title = titleEl.text().trim();
const href = titleEl.attr('href') || $el.find('a').first().attr('href') || '';
const desc = $el.find('.description, .teaser-text, p').first().text().trim();
// Try to extract signature count from text
const sigText = $el.text();
const sigMatch = sigText.match(/([\d,.]+)\s*(?:signatures?|Unterschriften|supporters?)/i);
const sigCount = sigMatch ? parseInt(sigMatch[1].replace(/[,.\s]/g, ''), 10) : 0;
if (title) {
const fullUrl = href.startsWith('http') ? href : `https://www.openpetition.eu${href}`;
const petId = href.split('/').filter(Boolean).pop() || `eu-${i}`;
items.push({
platform: 'openpetition_eu',
petition_id: petId,
title: title.substring(0, 500),
description: (desc || '').substring(0, 2000),
url: fullUrl,
signature_count: sigCount || 0,
signature_goal: null,
category: 'general',
creator: 'Unknown',
target: 'open',
created_date: null,
});
}
});
}
console.log(`[${AGENT}] OpenPetition EU: found ${items.length} petitions`);
} catch (err) {
console.error(`[${AGENT}] OpenPetition EU scrape error:`, err.message);
}
return items;
}
/**
* Scrape Decidim Barcelona (GraphQL API).
*
* @returns {Promise<Array<Object>>} Normalized petition objects
*/
async function scrapeDecidimBarcelona() {
const items = [];
try {
const query = `{
participatoryProcesses(filter: { withAnyScope: [] }) {
id
title { translation(locale: "en") }
description { translation(locale: "en") }
slug
startDate
endDate
publishedAt
}
}`;
const res = await fetchWithTimeout('https://www.decidim.barcelona/api', {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Accept': 'application/json',
},
body: JSON.stringify({ query }),
});
if (!res.ok) {
// Try fallback simpler query
const fallbackQuery = `{ participatoryProcesses { id slug title { translation(locale: "en") } } }`;
const res2 = await fetchWithTimeout('https://www.decidim.barcelona/api', {
method: 'POST',
headers: { 'Content-Type': 'application/json', 'Accept': 'application/json' },
body: JSON.stringify({ query: fallbackQuery }),
});
if (!res2.ok) return items;
const data2 = await res2.json();
const processes = data2?.data?.participatoryProcesses || [];
for (const proc of processes.slice(0, 20)) {
items.push({
platform: 'decidim',
petition_id: proc.id || proc.slug || `decidim-${items.length}`,
title: (proc.title?.translation || proc.slug || 'Untitled').substring(0, 500),
description: '',
url: `https://www.decidim.barcelona/processes/${proc.slug || proc.id}`,
signature_count: 0,
signature_goal: null,
category: 'participatory_process',
creator: 'Barcelona City Council',
target: 'active',
created_date: null,
});
}
console.log(`[${AGENT}] Decidim Barcelona (fallback): found ${items.length} processes`);
return items;
}
const data = await res.json();
const processes = data?.data?.participatoryProcesses || [];
for (const proc of processes.slice(0, 20)) {
const title = proc.title?.translation || proc.slug || 'Untitled';
const desc = proc.description?.translation || '';
// Strip HTML tags from description
const cleanDesc = desc.replace(/<[^>]*>/g, '').substring(0, 2000);
items.push({
platform: 'decidim',
petition_id: proc.id || proc.slug || `decidim-${items.length}`,
title: title.substring(0, 500),
description: cleanDesc,
url: `https://www.decidim.barcelona/processes/${proc.slug || proc.id}`,
signature_count: 0,
signature_goal: null,
category: 'participatory_process',
creator: 'Barcelona City Council',
target: proc.endDate ? (new Date(proc.endDate) > new Date() ? 'active' : 'ended') : 'active',
created_date: proc.startDate || proc.publishedAt || null,
});
}
console.log(`[${AGENT}] Decidim Barcelona: found ${items.length} processes`);
} catch (err) {
console.error(`[${AGENT}] Decidim Barcelona scrape error:`, err.message);
}
return items;
}
/**
* Scrape GovTrack Bills (JSON API).
*
* @returns {Promise<Array<Object>>} Normalized petition objects
*/
async function scrapeGovTrack() {
const items = [];
try {
const params = new URLSearchParams({
order_by: '-current_status_date',
limit: '20',
});
const url = `https://www.govtrack.us/api/v2/bill?${params}`;
const res = await fetchWithTimeout(url);
if (!res.ok) return items;
const json = await res.json();
const bills = json.objects || [];
for (const bill of bills.slice(0, 20)) {
// Use display_number (e.g. "S. 3315") as unique ID; fallback to congress+type+number
const billId = bill.display_number
|| `${bill.congress}-${bill.bill_type}-${bill.number}`
|| String(bill.id);
const title = bill.title || bill.title_without_number || 'Untitled Bill';
const status = bill.current_status || 'unknown';
const statusDate = bill.current_status_date || null;
const billUrl = bill.link || `https://www.govtrack.us/congress/bills/${bill.congress}/${bill.bill_type}${bill.number}`;
items.push({
platform: 'govtrack',
petition_id: String(billId),
title: title.substring(0, 500),
description: (bill.current_status_description || '').substring(0, 2000),
url: billUrl,
signature_count: bill.cosponsors ? bill.cosponsors.length : 0,
signature_goal: null,
category: 'legislation',
creator: bill.sponsor ? `${bill.sponsor.firstname} ${bill.sponsor.lastname}` : 'Congress',
target: status,
created_date: statusDate,
});
}
console.log(`[${AGENT}] GovTrack: found ${items.length} bills`);
} catch (err) {
console.error(`[${AGENT}] GovTrack scrape error:`, err.message);
}
return items;
}
/**
* Scrape MoveOn Trending (HTML scrape via cheerio).
*
* @returns {Promise<Array<Object>>} Normalized petition objects
*/
async function scrapeMoveOn() {
const items = [];
try {
const res = await fetchWithTimeout('https://sign.moveon.org/', {
headers: {
'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'en-US,en;q=0.9',
},
});
if (!res.ok) return items;
const html = await res.text();
const $ = cheerio.load(html);
// MoveOn lists trending petitions in various container patterns
const selectors = [
'.petition-card',
'.petition-item',
'.campaign-card',
'.petition',
'article',
'.card',
];
let petitionEls = $([]);
for (const sel of selectors) {
petitionEls = $(sel);
if (petitionEls.length > 0) break;
}
// Fallback: find links with /sign/ pattern
if (petitionEls.length === 0) {
const seenUrls = new Set();
$('a[href*="/sign/"], a[href*="/petition/"]').each((i, el) => {
if (items.length >= 20) return false;
const $el = $(el);
const href = $el.attr('href') || '';
const title = $el.text().trim();
// Skip nav/footer links and duplicates
if (!title || title.length < 10 || seenUrls.has(href)) return;
seenUrls.add(href);
const fullUrl = href.startsWith('http') ? href : `https://sign.moveon.org${href}`;
const petId = href.split('/').filter(Boolean).pop() || `moveon-${items.length}`;
items.push({
platform: 'moveon_trending',
petition_id: petId,
title: title.substring(0, 500),
description: '',
url: fullUrl,
signature_count: 0,
signature_goal: null,
category: 'campaign',
creator: 'MoveOn',
target: 'active',
created_date: null,
});
});
} else {
petitionEls.each((i, el) => {
if (i >= 20) return false;
const $el = $(el);
const titleEl = $el.find('h2, h3, h4, .title, a').first();
const title = titleEl.text().trim();
const href = titleEl.attr('href') || $el.find('a').first().attr('href') || '';
const desc = $el.find('.description, .summary, p').first().text().trim();
// Try to extract signature count
const text = $el.text();
const sigMatch = text.match(/([\d,]+)\s*(?:signatures?|signed|supporters?)/i);
const sigCount = sigMatch ? parseInt(sigMatch[1].replace(/,/g, ''), 10) : 0;
if (title && title.length > 5) {
const fullUrl = href.startsWith('http') ? href : `https://sign.moveon.org${href}`;
const petId = href.split('/').filter(Boolean).pop() || `moveon-${i}`;
items.push({
platform: 'moveon_trending',
petition_id: petId,
title: title.substring(0, 500),
description: (desc || '').substring(0, 2000),
url: fullUrl,
signature_count: sigCount || 0,
signature_goal: null,
category: 'campaign',
creator: 'MoveOn',
target: 'active',
created_date: null,
});
}
});
}
console.log(`[${AGENT}] MoveOn: found ${items.length} petitions`);
} catch (err) {
console.error(`[${AGENT}] MoveOn scrape error:`, err.message);
}
return items;
}
// ──────────────────────────────────────
// Gemini relevance scoring
// ──────────────────────────────────────
/**
* Score a batch of petitions for relevance to nonprofit advocacy / civic engagement
* using Gemini 2.0 Flash.
*
* @param {Array<Object>} petitions - Array of petition objects with title + description
* @returns {Promise<Object>} Map of petition_id -> { relevance_score, category }
*/
async function geminiScoreRelevance(petitions) {
const scores = {};
if (!petitions || petitions.length === 0) return scores;
// Batch petitions into groups for Gemini
const batch = petitions.slice(0, 25);
const petitionList = batch.map((p, i) =>
`${i + 1}. [${p.platform}] "${p.title}" — ${(p.description || '').substring(0, 200)}`
).join('\n');
try {
const prompt = `Score each petition/bill below for relevance to nonprofit advocacy, education policy, civic engagement, and social impact.
Petitions:
${petitionList}
Respond in EXACTLY this JSON format (no markdown, no code fences):
{"scores": [{"index": 1, "relevance": 0.0, "category": "topic"}, ...]}
Rules:
- "relevance" is 0.0 to 1.0 (0 = unrelated, 1 = directly about nonprofit advocacy/civic engagement)
- "category" is one of: advocacy, education, economic, healthcare, housing, environment, civil_rights, legislation, civic, general
- Score 0.8+ only for petitions directly about nonprofit advocacy, civic engagement, or social justice
- Score 0.5-0.7 for education policy, economic inequality, or public benefits
- Score 0.2-0.4 for broadly civic or social justice topics
- Score 0-0.1 for unrelated topics`;
const res = await fetch(GEMINI_URL, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
contents: [{ parts: [{ text: prompt }] }],
generationConfig: { temperature: 0.2, maxOutputTokens: 1000 },
}),
});
const data = await res.json();
const text = data?.candidates?.[0]?.content?.parts?.[0]?.text || '';
const jsonMatch = text.match(/\{[\s\S]*\}/);
if (jsonMatch) {
const parsed = JSON.parse(jsonMatch[0]);
const scoreList = parsed.scores || [];
for (const s of scoreList) {
const idx = (s.index || 0) - 1;
if (idx >= 0 && idx < batch.length) {
const pet = batch[idx];
scores[`${pet.platform}:${pet.petition_id}`] = {
relevance_score: Math.min(1, Math.max(0, Number(s.relevance) || 0)),
category: s.category || 'general',
};
}
}
}
} catch (err) {
console.error(`[${AGENT}] Gemini scoring error:`, err.message);
}
return scores;
}
// ──────────────────────────────────────
// Database upsert
// ──────────────────────────────────────
/**
* Upsert a petition into the trending_petitions table.
*
* @param {Object} petition - Normalized petition object
* @param {string} scratchBatch - Batch identifier for this scrape run
* @param {number} rank - Rank within this batch
* @returns {Promise<Object|null>} The upserted row or null on error
*/
async function upsertPetition(petition, scrapeBatch, rank) {
const pool = getPool();
try {
const { rows } = await pool.query(
`INSERT INTO trending_petitions
(platform, petition_id, title, description, target, signature_count,
signature_goal, category, url, creator, created_date,
is_climber, rank, scraped_at, scrape_batch)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, NOW(), $14)
ON CONFLICT (platform, petition_id) DO UPDATE SET
title = EXCLUDED.title,
description = EXCLUDED.description,
target = EXCLUDED.target,
signature_count = EXCLUDED.signature_count,
signature_goal = EXCLUDED.signature_goal,
category = EXCLUDED.category,
url = EXCLUDED.url,
creator = EXCLUDED.creator,
is_climber = CASE
WHEN trending_petitions.signature_count > 0
AND EXCLUDED.signature_count > trending_petitions.signature_count
THEN true ELSE false END,
growth_rate = CASE
WHEN trending_petitions.signature_count > 0
THEN EXCLUDED.signature_count - trending_petitions.signature_count
ELSE 0 END,
growth_pct = CASE
WHEN trending_petitions.signature_count > 0
THEN ROUND(((EXCLUDED.signature_count - trending_petitions.signature_count)::numeric
/ trending_petitions.signature_count) * 100, 2)
ELSE 0 END,
rank = EXCLUDED.rank,
scraped_at = NOW(),
scrape_batch = EXCLUDED.scrape_batch
RETURNING id, platform, petition_id, title, signature_count, is_climber`,
[
petition.platform,
petition.petition_id,
petition.title,
petition.description || null,
petition.target || null,
petition.signature_count || 0,
petition.signature_goal || null,
petition.category || 'general',
petition.url || null,
petition.creator || null,
petition.created_date || null,
false, // is_climber — computed in ON CONFLICT
rank,
scrapeBatch,
]
);
return rows[0] || null;
} catch (err) {
console.error(`[${AGENT}] DB upsert error for "${petition.title?.substring(0, 60)}":`, err.message);
return null;
}
}
// ──────────────────────────────────────
// Main skill handler
// ──────────────────────────────────────
/**
* Main scrape-petitions skill handler.
*
* Scrapes all registered petition platforms, scores relevance via Gemini,
* and upserts results into the trending_petitions table.
*
* @param {Object} [body={}] - Request body options
* @param {boolean} [body.score=true] - Whether to AI-score relevance
* @param {string} [body.platform] - Filter to a specific platform type
* @param {number} [body.max_per_platform] - Max petitions per platform (default 20)
* @returns {Promise<Object>} Scrape summary
*/
async function scrapePetitions(body = {}) {
const shouldScore = body.score !== false;
const platformFilter = body.platform || null;
const scrapeBatch = `petitions-${new Date().toISOString().replace(/[:.]/g, '-').substring(0, 19)}`;
console.log(`[${AGENT}] Starting petition scrape (batch: ${scrapeBatch})`);
// Map platform type to scraper function
const scrapers = {
uk_parliament: scrapeUKParliament,
openpetition_eu: scrapeOpenPetitionEU,
decidim: scrapeDecidimBarcelona,
govtrack: scrapeGovTrack,
moveon_trending: scrapeMoveOn,
};
let platforms = PETITION_PLATFORMS;
if (platformFilter) {
platforms = platforms.filter((p) => p.type === platformFilter);
}
let totalScraped = 0;
let totalUpserted = 0;
let totalFailed = 0;
let totalClimbers = 0;
const allPetitions = [];
const platformSummaries = [];
// Scrape each platform sequentially to avoid rate-limiting
for (const platform of platforms) {
console.log(`[${AGENT}] Scraping: ${platform.name}...`);
const scraperFn = scrapers[platform.type];
if (!scraperFn) {
console.warn(`[${AGENT}] No scraper for platform type: ${platform.type}`);
totalFailed++;
continue;
}
try {
const petitions = await scraperFn();
totalScraped += petitions.length;
allPetitions.push(...petitions);
platformSummaries.push({
platform: platform.type,
name: platform.name,
found: petitions.length,
});
} catch (err) {
console.error(`[${AGENT}] Failed to scrape ${platform.name}:`, err.message);
totalFailed++;
platformSummaries.push({
platform: platform.type,
name: platform.name,
found: 0,
error: err.message,
});
}
// Small delay between platforms
await new Promise((r) => setTimeout(r, 500));
}
// Score relevance with Gemini
let scores = {};
if (shouldScore && allPetitions.length > 0) {
console.log(`[${AGENT}] Scoring ${allPetitions.length} petitions with Gemini...`);
scores = await geminiScoreRelevance(allPetitions);
console.log(`[${AGENT}] Gemini scored ${Object.keys(scores).length} petitions`);
}
// Apply scores and upsert to DB
for (let i = 0; i < allPetitions.length; i++) {
const pet = allPetitions[i];
const scoreKey = `${pet.platform}:${pet.petition_id}`;
const scoreData = scores[scoreKey];
// Update category from Gemini scoring if available
if (scoreData && scoreData.category) {
pet.category = scoreData.category;
}
const result = await upsertPetition(pet, scrapeBatch, i + 1);
if (result) {
totalUpserted++;
if (result.is_climber) totalClimbers++;
}
}
// Build summary of top relevant petitions
const topRelevant = allPetitions
.map((p) => {
const key = `${p.platform}:${p.petition_id}`;
return {
platform: p.platform,
title: p.title,
signature_count: p.signature_count,
relevance: scores[key]?.relevance_score || 0,
category: scores[key]?.category || p.category,
url: p.url,
};
})
.filter((p) => p.relevance >= 0.3)
.sort((a, b) => b.relevance - a.relevance)
.slice(0, 10);
// Log audit action
await logAction({
agent: AGENT,
actionType: 'scrape',
platform: PLATFORM,
content: `Scraped ${totalScraped} petitions from ${platforms.length} platforms`,
responseData: {
batch: scrapeBatch,
platforms_scraped: platforms.length,
total_found: totalScraped,
total_upserted: totalUpserted,
total_failed: totalFailed,
total_climbers: totalClimbers,
top_relevant: topRelevant.slice(0, 5),
},
status: totalUpserted > 0 ? 'success' : (totalScraped > 0 ? 'success' : 'error'),
errorMessage: totalScraped === 0 ? 'No petitions scraped from any platform' : null,
});
const summary = {
batch: scrapeBatch,
platforms_scraped: platforms.length,
total_found: totalScraped,
total_upserted: totalUpserted,
total_failed: totalFailed,
total_climbers: totalClimbers,
platforms: platformSummaries,
top_relevant: topRelevant,
scored: shouldScore,
fetched_at: new Date().toISOString(),
};
console.log(`[${AGENT}] Petition scrape complete: ${totalScraped} found, ${totalUpserted} upserted, ${totalClimbers} climbers`);
return summary;
}
module.exports = { scrapePetitions };