← back to Ca Donations

ca-donations: public tier behind PUBLIC_TIER flag (default-off) — charitable open, political gated, k-anon aggregate rollups

b285cb40980eea081aa5695901e1018d9d11fc76 · 2026-08-24 11:50:42 -0700 · Steve Abrams

- per-path public allowlist in auth middleware, gated by PUBLIC_TIER (default off = byte-for-byte original fail-closed behavior)
- /api/political stays Basic-Auth gated even in public tier; only /api/stats,/api/orgs,/api/org/:ein,/api/grants + static open
- new /api/political/agg?by=recipient|employer|city|jurisdiction: grouped SUM/COUNT only, HAVING count(distinct donor_name)>=5 k-anon floor, no per-donor rows; behind SECOND flag PUBLIC_POLITICAL_AGG (default off)
- qWithTimeout() bounds the agg full-table scan (8s -> clean 503) so a public endpoint can't hang a connection
- X-Robots-Tag: noindex on /api/org/:ein; robots.txt (Disallow /api/); takedown.html; index.html public-mode banner + footer link

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Files touched

• M data/latest.json
• M lib/db.js
• M public/index.html
• A public/takedown.html
• M server.js

Diff

commit b285cb40980eea081aa5695901e1018d9d11fc76
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Mon Aug 24 11:50:42 2026 -0700

    ca-donations: public tier behind PUBLIC_TIER flag (default-off) — charitable open, political gated, k-anon aggregate rollups
    
    - per-path public allowlist in auth middleware, gated by PUBLIC_TIER (default off = byte-for-byte original fail-closed behavior)
    - /api/political stays Basic-Auth gated even in public tier; only /api/stats,/api/orgs,/api/org/:ein,/api/grants + static open
    - new /api/political/agg?by=recipient|employer|city|jurisdiction: grouped SUM/COUNT only, HAVING count(distinct donor_name)>=5 k-anon floor, no per-donor rows; behind SECOND flag PUBLIC_POLITICAL_AGG (default off)
    - qWithTimeout() bounds the agg full-table scan (8s -> clean 503) so a public endpoint can't hang a connection
    - X-Robots-Tag: noindex on /api/org/:ein; robots.txt (Disallow /api/); takedown.html; index.html public-mode banner + footer link
    
    Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
 data/latest.json     |  6 +++---
 lib/db.js            | 16 ++++++++++++++
 public/index.html    | 18 ++++++++++++++--
 public/takedown.html | 36 +++++++++++++++++++++++++++++++
 server.js            | 61 +++++++++++++++++++++++++++++++++++++++++++++++++---
 5 files changed, 129 insertions(+), 8 deletions(-)

diff --git a/data/latest.json b/data/latest.json
index 8caf0ba..355140e 100644
--- a/data/latest.json
+++ b/data/latest.json
@@ -2,7 +2,7 @@
   "service": "ca-donations-freshness",
   "verdict": "WARN",
   "status": "WARN",
-  "checked_at": "2026-08-24T18:26:31.139Z",
+  "checked_at": "2026-08-24T18:50:23.657Z",
   "findings": [
     {
       "source": "calaccess",
@@ -36,7 +36,7 @@
       "table": "charitable_orgs",
       "rows": 161285,
       "last_ok": "2026-08-21T15:11:50.911Z",
-      "age_days": 3.1,
+      "age_days": 3.2,
       "verdict": "PASS",
       "why": ""
     },
@@ -45,7 +45,7 @@
       "table": "charitable_orgs",
       "rows": 125,
       "last_ok": "2026-08-21T15:09:56.156Z",
-      "age_days": 3.1,
+      "age_days": 3.2,
       "verdict": "PASS",
       "why": ""
     }
diff --git a/lib/db.js b/lib/db.js
index 92280fd..198ae52 100644
--- a/lib/db.js
+++ b/lib/db.js
@@ -12,6 +12,22 @@ export async function q(text, params) {
   return res.rows;
 }
 
+// Run a query with a bounded statement_timeout on a dedicated client, so a heavy
+// aggregation (e.g. the public political rollup over the full table) cancels
+// cleanly instead of tying up a connection. The timeout is session-local and the
+// client is reset before returning to the pool.
+export async function qWithTimeout(text, params, ms = 8000) {
+  const client = await pool.connect();
+  try {
+    await client.query(`SET statement_timeout = ${Number(ms)}`);
+    const res = await client.query(text, params);
+    return res.rows;
+  } finally {
+    try { await client.query('SET statement_timeout = 0'); } catch (_) {}
+    client.release();
+  }
+}
+
 // Begin/finish an ingest_runs audit row. Returns { id, done(rowsIn, rowsUpsert, status, detail) }.
 export async function startRun(sourceSlug) {
   const [row] = await q(
diff --git a/public/index.html b/public/index.html
index 81fde10..1ff98d4 100644
--- a/public/index.html
+++ b/public/index.html
@@ -32,7 +32,10 @@
   .status-may-not-operate, .status-not-operating { color: #b4232a; }
   .empty { padding: 40px 18px; color: #667; text-align: center; }
   .pending { background: #fff6e0; border: 1px solid #f0d98a; color: #7a5c00; border-radius: 8px; padding: 16px; margin: 18px; }
+  .banner { display: none; padding: 9px 18px; background: #eaf1fb; color: #234; border-bottom: 1px solid #cfe0f5; font-size: 12px; }
+  .banner.show { display: block; }
   .note { font-size: 11px; color: #7a8797; padding: 0 18px 16px; }
+  .note a { color: #16324f; }
 </style>
 </head>
 <body>
@@ -40,6 +43,7 @@
   <h1>California Donations — Public Records</h1>
   <div class="sub">Political (donor-level, above the $100 CA itemization threshold) + Charitable (org &amp; foundation-grant records). Individual public-charity donors are not public.</div>
 </header>
+<div class="banner" id="banner">Public records from CA SoS / IRS / CA AG. Charitable &amp; foundation data shown. Political donor-level data is access-controlled.</div>
 <div class="stats" id="stats">loading…</div>
 <div class="tabs">
   <button data-tab="orgs" class="active">Charitable Orgs</button>
@@ -57,7 +61,7 @@
   </label>
 </div>
 <div id="view"></div>
-<div class="note">Sources: CAL-ACCESS (state) · FEC bulk (federal) · local NetFile/Socrata · CA AG Registry · IRS 990/990-PF · ProPublica. All $0 public data. Every value is clickable to refine.</div>
+<div class="note">Sources: CAL-ACCESS (state) · FEC bulk (federal) · local NetFile/Socrata · CA AG Registry · IRS 990/990-PF · ProPublica. All $0 public data. Every value is clickable to refine. · <a href="/takedown.html">Records &amp; removal requests</a></div>
 
 <script>
 const $ = (s) => document.querySelector(s);
@@ -97,6 +101,16 @@ function readURL(){ const u=new URLSearchParams(location.search); const t=u.get(
 function writeURL(push){ history[push?'pushState':'replaceState']({},'',qstr()); }
 window.qstr=qstr; window.readURL=readURL; window.writeURL=writeURL; window.hrefFor=hrefFor;
 
+// Public mode = /api/stats is reachable without creds but /api/political is not.
+async function detectPublicMode() {
+  try {
+    const [stats, pol] = await Promise.all([
+      fetch('/api/stats'), fetch('/api/political?limit=1'),
+    ]);
+    if (stats.ok && pol.status === 401) $('#banner').classList.add('show');
+  } catch (_) {}
+}
+
 async function loadStats() {
   const s = await (await fetch('/api/stats')).json();
   $('#stats').innerHTML =
@@ -182,7 +196,7 @@ document.querySelectorAll('.tabs button').forEach(b => b.onclick = () => {
 
 try { readURL(); } catch(_) {}
 window.addEventListener('popstate', () => { try{readURL();}catch(_){} document.querySelectorAll('.tabs button').forEach(b => b.classList.toggle('active', b.dataset.tab === tab)); syncControls(); render(); });
-syncControls(); loadStats(); render();
+syncControls(); loadStats(); detectPublicMode(); render();
 </script>
 </body>
 </html>
diff --git a/public/takedown.html b/public/takedown.html
new file mode 100644
index 0000000..38bd257
--- /dev/null
+++ b/public/takedown.html
@@ -0,0 +1,36 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="robots" content="noindex">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Records &amp; Removal — CA Donations</title>
+<style>
+  body { margin: 0; font: 15px/1.6 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif; color: #1a1a1a; background: #f6f7f9; }
+  header { padding: 14px 18px; background: #0f2540; color: #fff; }
+  header h1 { margin: 0; font-size: 17px; font-weight: 600; }
+  main { max-width: 720px; margin: 0 auto; padding: 24px 18px 60px; }
+  main h2 { font-size: 15px; margin: 26px 0 8px; }
+  a { color: #16324f; }
+  .back { display: inline-block; margin-top: 24px; font-size: 13px; }
+</style>
+</head>
+<body>
+<header><h1>Records &amp; Removal</h1></header>
+<main>
+  <p>This site indexes information that comes entirely from <strong>public records</strong> — the California Secretary of State (CAL-ACCESS), the Federal Election Commission, local NetFile/Socrata filings, the California Attorney General's Registry of Charitable Trusts, and IRS Forms 990/990-PF. We do not create these records; we mirror what governments and filers have already published.</p>
+
+  <h2>What is shown here</h2>
+  <p>Charitable and foundation data (organizations and grant lines) is shown openly. Individual political donor-level data is access-controlled and is not part of the public view. Individual public-charity donors are not published at all.</p>
+
+  <h2>Requesting a review or removal</h2>
+  <p>If a record here is inaccurate, out of date, or you believe it should be reviewed for removal, email us with the specific record (a name, EIN, or the page URL) and what you would like corrected. We will review each request individually.</p>
+  <p>Because the underlying data is a matter of public record at its source, a correction here does not change the government filing itself — for that you would contact the originating agency. We can, however, review what this site displays.</p>
+
+  <h2>Contact</h2>
+  <p><a href="mailto:info@agentabrams.com?subject=CA%20Donations%20record%20review">info@agentabrams.com</a></p>
+
+  <a class="back" href="/">&larr; Back to the records</a>
+</main>
+</body>
+</html>
diff --git a/server.js b/server.js
index ceffa40..396e495 100644
--- a/server.js
+++ b/server.js
@@ -1,16 +1,36 @@
 // ca-donations — Basic-Auth searchable data product over CA donation public records.
 // Two families: charitable (orgs + grants) and political (donor-level contributions).
 import express from 'express';
-import { q } from './lib/db.js';
+import { q, qWithTimeout } from './lib/db.js';
 
 const app = express();
 const PORT = process.env.PORT || 9926;
 const USER = process.env.BASIC_AUTH_USER || 'admin';
 const PASS = process.env.BASIC_AUTH_PASS || 'DW2024!';
 
-// --- Basic Auth gate (401 = healthy). /healthz is open for smoke checks. ---
+// Public tier is OFF by default — when unset/'0' the server is byte-for-byte the
+// original fail-closed Basic-Auth app. Political AGG is a SECOND independent flag:
+// even with the tier on, aggregate political stays dark until it too is enabled.
+const PUBLIC_TIER = process.env.PUBLIC_TIER === '1';
+const PUBLIC_POLITICAL_AGG = process.env.PUBLIC_POLITICAL_AGG === '1';
+
+// When the public tier is on, ONLY these read-only charitable/org paths open to
+// the public. /api/political (raw donor rows) NEVER opens. Static public/ opens
+// via the isPublicPath check below. Everything else stays Basic-Auth gated.
+const publicApiPath = (path) =>
+  path === '/api/stats' ||
+  path === '/api/orgs' ||
+  path === '/api/grants' ||
+  path.startsWith('/api/org/') ||
+  (PUBLIC_POLITICAL_AGG && path === '/api/political/agg');
+
+const isPublicPath = (path) =>
+  path === '/healthz' ||
+  (PUBLIC_TIER && (publicApiPath(path) || !path.startsWith('/api/')));
+
+// --- Basic Auth gate (401 = healthy). Open paths per the tier allowlist. ---
 app.use((req, res, next) => {
-  if (req.path === '/healthz') return next();
+  if (isPublicPath(req.path)) return next();
   const h = req.headers.authorization || '';
   const [, b64] = h.split(' ');
   if (b64) {
@@ -81,6 +101,7 @@ app.get('/api/orgs', async (req, res) => {
 // Org detail + its grants (grantor and grantee sides).
 app.get('/api/org/:ein', async (req, res) => {
   try {
+    res.set('X-Robots-Tag', 'noindex');
     const [org] = await q('SELECT * FROM charitable_orgs WHERE ein=$1', [req.params.ein]);
     const grantsOut = await q('SELECT * FROM charitable_grants WHERE grantor_ein=$1 ORDER BY tax_year DESC, amount DESC LIMIT 200', [req.params.ein]);
     res.json({ org: org || null, grants_out: grantsOut });
@@ -122,6 +143,40 @@ app.get('/api/political', async (req, res) => {
   } catch (e) { res.status(500).json({ error: e.message }); }
 });
 
+// Aggregate-only political rollups — NO per-donor rows ever leave here. Each row
+// is a group SUM(amount) + COUNT(*), and the HAVING floor drops any group with
+// fewer than 5 distinct donors so a group can never single out one person.
+// Gated behind BOTH flags: only reachable when PUBLIC_TIER + PUBLIC_POLITICAL_AGG.
+const AGG_GROUP = {
+  recipient:    'recipient_name',
+  employer:     'donor_employer',
+  city:         'donor_city',
+  jurisdiction: 'jurisdiction',
+};
+app.get('/api/political/agg', async (req, res) => {
+  try {
+    const col = AGG_GROUP[req.query.by];
+    if (!col) return res.status(400).json({ error: 'by must be one of: ' + Object.keys(AGG_GROUP).join('|') });
+    const limit = Math.min(+req.query.limit || 100, 500);
+    // Bounded — a full-table GROUP BY over ~15M rows must cancel cleanly, not hang.
+    const rows = await qWithTimeout(
+      `SELECT ${col} AS group_key,
+              sum(amount)::numeric(14,2)     AS total_amount,
+              count(*)::int                  AS contribution_count,
+              count(distinct donor_name)::int AS distinct_donors
+       FROM political_contributions
+       WHERE ${POL_CLEAN} AND ${col} IS NOT NULL AND ${col} <> ''
+       GROUP BY ${col}
+       HAVING count(distinct donor_name) >= 5
+       ORDER BY sum(amount) DESC NULLS LAST
+       LIMIT $1`, [limit], 8000);
+    res.json({ by: req.query.by, k_anonymity_floor: 5, rows });
+  } catch (e) {
+    if (e.code === '57014') return res.status(503).json({ error: 'aggregate timed out' });
+    res.status(500).json({ error: e.message });
+  }
+});
+
 app.use(express.static('public'));
 
 app.listen(PORT, () => console.log(`ca-donations on :${PORT}`));

← 1068eb4 auto-data-snapshot: 2026-08-24T11:44:27 (2 data files) — dat  ·  back to Ca Donations  ·  ca-donations: harden public gate — strict positive allowlist 80ce7b1 →