← back to Pitch Guard
Fix non-deterministic followup-candidate scanner: single-flight lock + serve cache on reload, retry throttled Gmail checks instead of silently dropping (over-include w/ 'uncertain' flag), full paginated sent scan (80 contacts). Count now stable across reloads (was 11→10→6)
0d9ee55d614e2ca3d3557b36127b255378b98d44 · 2026-06-16 14:12:52 -0700 · SteveStudio2
Files touched
M public/index.htmlM server.js
Diff
commit 0d9ee55d614e2ca3d3557b36127b255378b98d44
Author: SteveStudio2 <steve@designerwallcoverings.com>
Date: Tue Jun 16 14:12:52 2026 -0700
Fix non-deterministic followup-candidate scanner: single-flight lock + serve cache on reload, retry throttled Gmail checks instead of silently dropping (over-include w/ 'uncertain' flag), full paginated sent scan (80 contacts). Count now stable across reloads (was 11→10→6)
---
public/index.html | 2 +-
server.js | 90 ++++++++++++++++++++++++++++++++++++++-----------------
2 files changed, 63 insertions(+), 29 deletions(-)
diff --git a/public/index.html b/public/index.html
index 7743934..7bd5d31 100644
--- a/public/index.html
+++ b/public/index.html
@@ -464,7 +464,7 @@ function candidateSectionHTML(){
else body=(candScannedAt?'<div class="vgroup-note">Auto-scans daily · last scan '+fmtDate(candScannedAt)+' — review, then ✎ Draft it.</div>':'')+CANDIDATES.map(c=>
'<div class="cand" data-email="'+esc(c.contact)+'">'+
'<button class="cand-draft" data-draft-cand="'+esc(c.contact)+'" title="Create the followup draft">✎ Draft it</button>'+
- '<div class="who">'+esc(c.contactName)+'</div>'+
+ '<div class="who">'+esc(c.contactName)+(c.uncertain?' <span title="Reply/draft check was throttled — shown so it is not silently dropped; verify the thread before sending" style="color:#b8860b;font-size:11px">⚠ verify</span>':'')+'</div>'+
'<div class="subj">'+esc(c.subject)+'</div>'+
'<div class="meta">'+esc(c.contact)+' · '+c.daysSince+'d no reply</div>'+
'</div>').join('');
diff --git a/server.js b/server.js
index e4d8a0f..6edeb15 100644
--- a/server.js
+++ b/server.js
@@ -120,6 +120,18 @@ function cacheGet(k) { const e = cache.get(k); if (e && Date.now() - e.t < TTL)
function cacheSet(k, v) { cache.set(k, { t: Date.now(), v }); }
const sleep = ms => new Promise(r => setTimeout(r, ms));
+// Gmail (via George) throttles bursts → a transient 429/5xx must NOT be swallowed
+// into a silently-dropped candidate (that was the root of the fluctuating count).
+// Retry with backoff; only a hard, repeated failure propagates to the caller.
+async function georgeRetry(p, opts, tries = 3) {
+ let lastErr;
+ for (let i = 0; i < tries; i++) {
+ try { return await george(p, opts); }
+ catch (e) { lastErr = e; await sleep(500 * (i + 1)); }
+ }
+ throw lastErr;
+}
+
// ── persistent sent-letter ledger (data/sent-log.json) ─────────────────────
// Steve's ask: keep the dates + the actual letters we sent to each person,
// durably — so the record survives restarts and any Gmail-search lag.
@@ -187,10 +199,12 @@ let candCache = null;
const CAND_TTL = 20 * 60 * 1000;
async function runCandidateScan() {
- // our sent mail, 5–35 days old — the window where a followup comes due
- let sent = [], pageToken;
- for (let pg = 0; pg < 2; pg++) {
- const out = await george('/api/messages', { query: { account: ACCOUNT, q: 'in:sent older_than:5d newer_than:35d', maxResults: 100, pageToken } });
+ // our sent mail, 5–35 days old — the window where a followup comes due.
+ // Fully paginate (capped) with retry so the candidate universe is COMPLETE
+ // and stable rather than a throttle-truncated slice that varies per scan.
+ let sent = [], pageToken, SENT_PAGES = 4; // 4 × 100 = up to 400 sent msgs
+ for (let pg = 0; pg < SENT_PAGES; pg++) {
+ const out = await georgeRetry('/api/messages', { query: { account: ACCOUNT, q: 'in:sent older_than:5d newer_than:35d', maxResults: 100, pageToken } });
sent = sent.concat(out.messages || []);
pageToken = out.nextPageToken;
if (!pageToken) break;
@@ -205,47 +219,67 @@ async function runCandidateScan() {
const cur = byContact.get(c);
if (!cur || ts > cur.ts) byContact.set(c, { ts, msg: m });
}
- const contacts = [...byContact.entries()].sort((a, b) => b[1].ts - a[1].ts).slice(0, 55);
+ const contacts = [...byContact.entries()].sort((a, b) => b[1].ts - a[1].ts).slice(0, 80);
const candidates = [];
for (const [contact, info] of contacts) {
const daysSince = Math.floor((Date.now() - info.ts) / 86400000);
if (daysSince < 5) continue;
if (classifyRecipient(contact) === 'vendor') continue;
+ // did the client reply after our last message? (from: + after:YYYY/MM/DD)
+ // is a followup draft to this contact already pending?
+ // A throttled check must NOT silently drop the candidate (that caused the
+ // 11→6 fluctuation) — retry, and if it still fails, INCLUDE it flagged
+ // `uncertain` so Steve sees it (over-include is safe: the risk-gate + his
+ // review catch a false positive; a silently-dropped real lead is lost revenue).
+ const sd = new Date(info.ts);
+ const after = `${sd.getFullYear()}/${sd.getMonth() + 1}/${sd.getDate()}`;
+ let uncertain = false;
try {
- // did the client reply after our last message? (from: + after:YYYY/MM/DD)
- const sd = new Date(info.ts);
- const after = `${sd.getFullYear()}/${sd.getMonth() + 1}/${sd.getDate()}`;
- const reply = await george('/api/search', { query: { account: ACCOUNT, q: `from:${contact} after:${after}`, maxResults: 2 } });
- if ((reply.messages || []).some(x => !x.error)) continue; // they replied → not ghosted
- // is a followup draft to this contact already pending?
- const dr = await george('/api/search', { query: { account: ACCOUNT, q: `in:draft to:${contact}`, maxResults: 2 } });
- if ((dr.messages || []).some(x => !x.error)) continue; // already drafted
- candidates.push({
- threadId: info.msg.threadId,
- subject: info.msg.subject || '(no subject)',
- contact, contactName: displayName(info.msg.to) || contact,
- lastDate: info.msg.date, daysSince,
- });
- } catch (e) {}
+ const reply = await georgeRetry('/api/search', { query: { account: ACCOUNT, q: `from:${contact} after:${after}`, maxResults: 2 } });
+ if ((reply.messages || []).some(x => !x.error)) { await sleep(150); continue; } // replied → not ghosted
+ } catch (e) { uncertain = true; }
+ try {
+ const dr = await georgeRetry('/api/search', { query: { account: ACCOUNT, q: `in:draft to:${contact}`, maxResults: 2 } });
+ if ((dr.messages || []).some(x => !x.error)) { await sleep(150); continue; } // already drafted
+ } catch (e) { uncertain = true; }
+ candidates.push({
+ threadId: info.msg.threadId,
+ subject: info.msg.subject || '(no subject)',
+ contact, contactName: displayName(info.msg.to) || contact,
+ lastDate: info.msg.date, daysSince, uncertain,
+ });
await sleep(150);
}
candidates.sort((a, b) => b.daysSince - a.daysSince);
- const payload = { count: candidates.length, scanned: contacts.length, candidates, scannedAt: new Date().toISOString() };
+ const payload = { count: candidates.length, scanned: contacts.length,
+ uncertainCount: candidates.filter(c => c.uncertain).length,
+ candidates, scannedAt: new Date().toISOString() };
candCache = { t: Date.now(), v: payload };
return payload;
}
+// Single-flight: a fresh cache is served directly; otherwise ALL concurrent
+// callers (page auto-load + manual rescans + the daily timer) coalesce onto the
+// ONE in-flight scan instead of each firing its own — which is what made the
+// count differ per reload (last-writer-wins on candCache). Reloads now read the
+// cache and the number is stable until the 20-min TTL or an explicit ?fresh=1.
+let candScanInFlight = null;
+function getCandidates(fresh) {
+ if (!fresh && candCache && Date.now() - candCache.t < CAND_TTL)
+ return Promise.resolve({ ...candCache.v, cached: true });
+ if (candScanInFlight) return candScanInFlight;
+ candScanInFlight = runCandidateScan().finally(() => { candScanInFlight = null; });
+ return candScanInFlight;
+}
+
app.get('/api/followup-candidates', async (req, res) => {
- try {
- if (candCache && !req.query.fresh && Date.now() - candCache.t < CAND_TTL)
- return res.json({ ...candCache.v, cached: true });
- res.json(await runCandidateScan());
- } catch (e) { res.status(e.status || 500).json({ error: e.message }); }
+ try { res.json(await getCandidates(!!req.query.fresh)); }
+ catch (e) { res.status(e.status || 500).json({ error: e.message }); }
});
// Auto-scan: warm the candidate list ~30s after startup, then once a day.
-setTimeout(() => { runCandidateScan().catch(e => console.warn('[followup auto-scan] startup:', e.message)); }, 30_000);
-setInterval(() => { runCandidateScan().catch(e => console.warn('[followup auto-scan] daily:', e.message)); }, 24 * 60 * 60 * 1000);
+setTimeout(() => { getCandidates(true).catch(e => console.warn('[followup auto-scan] startup:', e.message)); }, 30_000);
+setInterval(() => { getCandidates(true).catch(e => console.warn('[followup auto-scan] daily:', e.message)); }, 24 * 60 * 60 * 1000);
// ── API: full message (works for drafts too) ──────────────────────────────
app.get('/api/message/:id', async (req, res) => {
← bd41631 pitch-guard: persist dates+letters sent per person (durable
·
back to Pitch Guard
·
Refine: never bother clients who already transacted or were 7308e45 →