← back to Grant
news: replace AI-fabricated discovery with real Federal Register API data
f2decb07887ec00811ed0d65b730fb4d72435dec · 2026-05-30 08:35:54 -0700 · Steve
Drop the Gemini path that asked an LLM to invent "realistic" headlines,
"plausible" URLs, and "plausible" journalist names. /api/news/discover now
fetches real, published documents live from the free, open Federal Register
API (no key) and maps them to the existing news_items shape — outlet=agency,
summary=abstract, url=verifiable federalregister.gov link, source_type from
FR doc type. Org AI keywords drive a relevance term-search so results stay
on-mission. Response contract ({ discovered, news }) and DB row shape are
unchanged, so NewsTab keeps working. Adds "Federal Register" attribution
(author_name + tag + response.source). New lib/federalRegister.ts client.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
M app/api/news/discover/route.tsA lib/federalRegister.ts
Diff
commit f2decb07887ec00811ed0d65b730fb4d72435dec
Author: Steve <steve@designerwallcoverings.com>
Date: Sat May 30 08:35:54 2026 -0700
news: replace AI-fabricated discovery with real Federal Register API data
Drop the Gemini path that asked an LLM to invent "realistic" headlines,
"plausible" URLs, and "plausible" journalist names. /api/news/discover now
fetches real, published documents live from the free, open Federal Register
API (no key) and maps them to the existing news_items shape — outlet=agency,
summary=abstract, url=verifiable federalregister.gov link, source_type from
FR doc type. Org AI keywords drive a relevance term-search so results stay
on-mission. Response contract ({ discovered, news }) and DB row shape are
unchanged, so NewsTab keeps working. Adds "Federal Register" attribution
(author_name + tag + response.source). New lib/federalRegister.ts client.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
app/api/news/discover/route.ts | 127 ++++++++++++----------------
lib/federalRegister.ts | 184 +++++++++++++++++++++++++++++++++++++++++
2 files changed, 236 insertions(+), 75 deletions(-)
diff --git a/app/api/news/discover/route.ts b/app/api/news/discover/route.ts
index e51004c..ebbd36b 100644
--- a/app/api/news/discover/route.ts
+++ b/app/api/news/discover/route.ts
@@ -2,20 +2,22 @@ import { NextRequest, NextResponse } from 'next/server';
import { query } from '@/lib/db';
import { verifyAuthWithOrg, resolveOrgId } from '@/lib/auth';
import { auditLog } from '@/lib/audit';
-import { callGemini } from '@/lib/gemini';
-
-type NewsItem = {
- headline?: string;
- outlet?: string;
- url?: string;
- published_at?: string;
- summary?: string;
- tags?: string[];
- relevance_score?: number;
- author_name?: string;
- source_type?: string;
-};
+import { fetchFederalRegisterNews } from '@/lib/federalRegister';
+/*
+ * POST /api/news/discover
+ *
+ * 2026-05-30: replaced the AI-fabricated news path (which asked Gemini to
+ * invent "realistic" headlines, "plausible" URLs, and "plausible" journalist
+ * names) with REAL data pulled live from the free, open Federal Register API
+ * (https://www.federalregister.gov/api/v1/documents.json — no API key).
+ *
+ * Every inserted row is now a real, published federal document with a
+ * verifiable html_url on federalregister.gov. The org's AI keywords drive a
+ * relevance term-search so results stay on-mission. The response contract
+ * ({ discovered, news }) and the news_items row shape are unchanged, so the
+ * NewsTab frontend keeps working without modification.
+ */
export async function POST(request: NextRequest) {
const session = verifyAuthWithOrg(request);
if (!session) return NextResponse.json({ error: 'Unauthorized' }, { status: 401 });
@@ -25,7 +27,7 @@ export async function POST(request: NextRequest) {
const orgId = await resolveOrgId(session);
if (!orgId) return NextResponse.json({ error: 'No organization found' }, { status: 400 });
const orgRes = await query(
- 'SELECT id, name, ai_mission, ai_focus_areas, ai_keywords, ai_org_summary FROM organizations WHERE id = $1',
+ 'SELECT id, name, ai_focus_areas, ai_keywords FROM organizations WHERE id = $1',
[orgId],
);
if (orgRes.rows.length === 0) {
@@ -33,60 +35,32 @@ export async function POST(request: NextRequest) {
}
const org = orgRes.rows[0];
- // Get existing headlines to avoid duplicates
- const existingRes = await query(
- 'SELECT headline FROM news_items WHERE org_id = $1',
- [org.id],
- );
- const existingHeadlines = existingRes.rows.map((r) => r.headline);
-
- const prompt = `You are a news research assistant for a nonprofit organization. Find 8-12 REAL and RECENT news articles relevant to this organization's mission.
-
-Organization: ${org.name}
-Mission: ${org.ai_mission}
-Focus Areas: ${(org.ai_focus_areas || []).join(', ')}
-Keywords: ${(org.ai_keywords || []).join(', ')}
-
-${existingHeadlines.length > 0 ? `Already tracked (DO NOT repeat): ${existingHeadlines.slice(0, 20).join('; ')}` : ''}
+ // Build the keyword set the FR term-search runs on: org keywords first,
+ // then focus areas as a fallback for relevance.
+ const keywords: string[] = [
+ ...(Array.isArray(org.ai_keywords) ? org.ai_keywords : []),
+ ...(Array.isArray(org.ai_focus_areas) ? org.ai_focus_areas : []),
+ ].filter((k) => typeof k === 'string' && k.trim());
-Focus on news from late 2025 and early 2026 about:
-- Student debt policy changes, forgiveness programs, PSLF updates
-- Higher education funding and reform
-- FAFSA changes, Pell Grant updates
-- For-profit college accountability
-- Consumer financial protection related to student loans
-- Congressional actions on education funding
-- State-level student debt initiatives
-
-Return ONLY a JSON array (no markdown, no code fences) of news objects with:
-- headline: string (realistic news headline)
-- outlet: string (real news outlet name like NPR, The Washington Post, Inside Higher Ed, The Hill, Reuters, CNBC, Politico, etc.)
-- url: string (plausible URL at that outlet)
-- published_at: string (ISO date, recent dates in Jan-Feb 2026)
-- summary: string (2-3 sentence summary of the article)
-- tags: string[] (3-5 relevant tags)
-- relevance_score: number (0-1 score of relevance to the org)
-- author_name: string (plausible journalist name)
-- source_type: string (one of: "google_news", "press_release", "policy_brief", "academic", "industry")`;
-
- // 2026-05-04 (architect-reviewer): migrated to lib/gemini.ts wrapper.
- // The wrapper centralizes timeout / fence-strip / parse / 503-503-502-504
- // status mapping; this handler now only owns prompt + persistence.
- const result = await callGemini<NewsItem[]>({ prompt, maxTokens: 8192 });
- if (!result.ok) {
- console.error('[news/discover] gemini failed:', result.reason, result.detail);
+ const fr = await fetchFederalRegisterNews({ keywords, perPage: 25 });
+ if (!fr.ok) {
+ console.error('[news/discover] Federal Register fetch failed:', fr.reason, fr.detail);
return NextResponse.json(
- { error: result.reason === 'no_key' ? 'AI service not configured' : 'AI service error' },
- { status: result.status },
+ { error: `Federal Register source unavailable (${fr.reason})` },
+ { status: fr.reason === 'http' ? 502 : 503 },
);
}
- const newsItems = result.data;
- if (!Array.isArray(newsItems)) {
- return NextResponse.json({ error: 'AI returned invalid format' }, { status: 500 });
- }
+
+ // Dedupe against headlines already tracked for this org.
+ const existingRes = await query(
+ 'SELECT headline FROM news_items WHERE org_id = $1',
+ [org.id],
+ );
+ const existing = new Set(existingRes.rows.map((r) => (r.headline || '').trim()));
const inserted = [];
- for (const n of newsItems) {
+ for (const n of fr.items) {
+ if (existing.has(n.headline.trim())) continue;
try {
const result = await query(
`INSERT INTO news_items (org_id, headline, outlet, url, published_at, summary, tags, relevance_score, author_name, source_type)
@@ -94,32 +68,35 @@ Return ONLY a JSON array (no markdown, no code fences) of news objects with:
RETURNING id, org_id, headline, outlet, url, published_at, summary, tags, relevance_score, author_name, source_type, created_at, updated_at`,
[
org.id,
- n.headline || 'Untitled',
- n.outlet || 'Unknown',
- n.url || null,
- n.published_at || null,
- n.summary || null,
- n.tags || [],
- n.relevance_score != null ? n.relevance_score : 0.5,
- n.author_name || null,
- n.source_type || 'google_news',
+ n.headline,
+ n.outlet,
+ n.url,
+ n.published_at,
+ n.summary,
+ n.tags,
+ n.relevance_score,
+ n.author_name,
+ n.source_type,
],
);
inserted.push(result.rows[0]);
+ existing.add(n.headline.trim());
} catch (insertErr) {
console.error('[news/discover] Insert error:', n.headline, insertErr);
}
}
- // 2026-05-05 (architect P2-3): audit AI call.
- auditLog('news.ai_discovered', 'news_item', null, org.id, user, {
+ auditLog('news.fr_discovered', 'news_item', null, org.id, user, {
+ source: 'federal_register',
+ term: fr.term,
+ fetched_count: fr.items.length,
+ total_available: fr.count,
discovered_count: inserted.length,
- input_tokens: result.inputTokens,
- output_tokens: result.outputTokens,
}, request.headers.get('x-forwarded-for') || undefined);
return NextResponse.json({
discovered: inserted.length,
+ source: 'Federal Register',
news: inserted,
});
} catch (err) {
diff --git a/lib/federalRegister.ts b/lib/federalRegister.ts
new file mode 100644
index 0000000..7aa2798
--- /dev/null
+++ b/lib/federalRegister.ts
@@ -0,0 +1,184 @@
+/**
+ * lib/federalRegister.ts
+ *
+ * Real-data client for the free, open Federal Register API
+ * (https://www.federalregister.gov/api/v1/documents.json — no API key required).
+ *
+ * Replaces the prior AI-fabricated "news" path that asked an LLM to invent
+ * "realistic" headlines, "plausible" URLs, and "plausible" journalist names.
+ * Every item this returns is a real, published Federal Register document with a
+ * verifiable html_url on federalregister.gov.
+ */
+
+const FR_BASE = 'https://www.federalregister.gov/api/v1/documents.json';
+
+/** Shape of a single agency entry inside an FR document. */
+interface FRAgency {
+ raw_name?: string;
+ name?: string;
+}
+
+/** Subset of an FR document we request via fields[]. */
+interface FRDocument {
+ title?: string;
+ abstract?: string;
+ publication_date?: string;
+ html_url?: string;
+ agencies?: FRAgency[];
+ type?: string;
+ document_number?: string;
+}
+
+interface FRResponse {
+ count?: number;
+ results?: FRDocument[];
+}
+
+/** The normalized news item shape the rest of the app (DB + frontend) consumes. */
+export interface NewsItem {
+ headline: string;
+ outlet: string;
+ url: string | null;
+ published_at: string | null;
+ summary: string | null;
+ tags: string[];
+ relevance_score: number;
+ author_name: string | null;
+ source_type: string;
+}
+
+export type FRFetchResult =
+ | { ok: true; items: NewsItem[]; count: number; term: string | null }
+ | { ok: false; reason: 'network' | 'http' | 'parse'; detail: string };
+
+/** FR document type → the app's source_type taxonomy (kept frontend-compatible). */
+function mapSourceType(frType?: string): string {
+ switch ((frType || '').toLowerCase()) {
+ case 'rule':
+ case 'proposed rule':
+ return 'policy_brief';
+ case 'notice':
+ return 'press_release';
+ case 'presidential document':
+ return 'industry';
+ default:
+ return 'google_news';
+ }
+}
+
+/** Best human-readable agency name for the "outlet" slot. */
+function primaryAgency(agencies?: FRAgency[]): string {
+ if (!agencies || agencies.length === 0) return 'Federal Register';
+ const a = agencies[0];
+ return a.name || a.raw_name || 'Federal Register';
+}
+
+/**
+ * Lightweight relevance score in [0,1]. The FR term-search already filters to
+ * matching docs, so a real (non-fabricated) heuristic just nudges items whose
+ * title/abstract actually contain the org's keywords higher. Defaults to 0.5
+ * when no keywords are supplied.
+ */
+function scoreRelevance(doc: FRDocument, keywords: string[]): number {
+ if (keywords.length === 0) return 0.5;
+ const hay = `${doc.title || ''} ${doc.abstract || ''}`.toLowerCase();
+ let hits = 0;
+ for (const kw of keywords) {
+ const k = kw.trim().toLowerCase();
+ if (k && hay.includes(k)) hits++;
+ }
+ if (hits === 0) return 0.5;
+ // 1 hit → 0.7, scaling up toward 0.95 with more keyword matches.
+ return Math.min(0.95, 0.6 + 0.1 * hits + 0.05);
+}
+
+/** Build the tag list from the FR doc type + matched org keywords. */
+function buildTags(doc: FRDocument, keywords: string[]): string[] {
+ const tags = new Set<string>();
+ if (doc.type) tags.add(doc.type);
+ tags.add('Federal Register');
+ const hay = `${doc.title || ''} ${doc.abstract || ''}`.toLowerCase();
+ for (const kw of keywords) {
+ const k = kw.trim();
+ if (k && hay.includes(k.toLowerCase())) tags.add(k);
+ }
+ return Array.from(tags).slice(0, 5);
+}
+
+/**
+ * Fetch real, recent Federal Register documents and map them into the app's
+ * news-item shape.
+ *
+ * @param opts.keywords org keywords; the first few drive an FR term search
+ * and also feed relevance scoring + tagging.
+ * @param opts.perPage how many docs to request (1-100). Default 20.
+ * @param opts.timeoutMs network timeout. Default 15000.
+ */
+export async function fetchFederalRegisterNews(opts: {
+ keywords?: string[];
+ perPage?: number;
+ timeoutMs?: number;
+} = {}): Promise<FRFetchResult> {
+ const keywords = (opts.keywords || []).filter((k) => typeof k === 'string' && k.trim());
+ const perPage = Math.min(Math.max(opts.perPage ?? 20, 1), 100);
+ const timeoutMs = opts.timeoutMs ?? 15000;
+
+ // Use up to 4 keywords joined as a single OR-ish term search. The FR API
+ // treats `conditions[term]` as a relevance full-text query.
+ const term = keywords.length > 0 ? keywords.slice(0, 4).join(' ') : null;
+
+ const params = new URLSearchParams();
+ params.append('conditions[type][]', 'RULE');
+ params.append('conditions[type][]', 'NOTICE');
+ params.append('conditions[type][]', 'PRORULE');
+ if (term) params.append('conditions[term]', term);
+ params.append('per_page', String(perPage));
+ params.append('order', term ? 'relevance' : 'newest');
+ for (const f of ['title', 'abstract', 'publication_date', 'html_url', 'agencies', 'type', 'document_number']) {
+ params.append('fields[]', f);
+ }
+
+ const url = `${FR_BASE}?${params.toString()}`;
+
+ let res: Response;
+ const controller = new AbortController();
+ const timer = setTimeout(() => controller.abort(), timeoutMs);
+ try {
+ res = await fetch(url, {
+ signal: controller.signal,
+ headers: { Accept: 'application/json', 'User-Agent': 'Grant-Nonprofit-NewsMonitor/1.0' },
+ });
+ } catch (err) {
+ clearTimeout(timer);
+ return { ok: false, reason: 'network', detail: (err as Error).message };
+ }
+ clearTimeout(timer);
+
+ if (!res.ok) {
+ return { ok: false, reason: 'http', detail: `Federal Register API returned ${res.status}` };
+ }
+
+ let body: FRResponse;
+ try {
+ body = (await res.json()) as FRResponse;
+ } catch (err) {
+ return { ok: false, reason: 'parse', detail: (err as Error).message };
+ }
+
+ const results = Array.isArray(body.results) ? body.results : [];
+ const items: NewsItem[] = results
+ .filter((d) => d.title && d.html_url)
+ .map((d) => ({
+ headline: d.title!.trim(),
+ outlet: primaryAgency(d.agencies),
+ url: d.html_url || null,
+ published_at: d.publication_date || null,
+ summary: d.abstract?.trim() || null,
+ tags: buildTags(d, keywords),
+ relevance_score: scoreRelevance(d, keywords),
+ author_name: 'Federal Register', // source attribution; FR docs have no byline
+ source_type: mapSourceType(d.type),
+ }));
+
+ return { ok: true, items, count: body.count ?? items.length, term };
+}
← 23aa243 Add grant-competitor peer survey
·
back to Grant
·
Add 6 Grant public-landing mockups 328ab89 →