[object Object]

← back to Ken

Ken reconcile-canary: retry transient PG restart errors (57P03/ENOENT/ECONNREFUSED) before exit 2; log error code (TK-11999)

a2449aca11edf1bef6ef4be7477db2dd4339a861 · 2026-09-26 09:58:50 -0700 · Steve Abrams

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01GEE4jWy9RBmz5QnR2xcvno

Files touched

Diff

commit a2449aca11edf1bef6ef4be7477db2dd4339a861
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sat Sep 26 09:58:50 2026 -0700

    Ken reconcile-canary: retry transient PG restart errors (57P03/ENOENT/ECONNREFUSED) before exit 2; log error code (TK-11999)
    
    Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
    Claude-Session: https://claude.ai/code/session_01GEE4jWy9RBmz5QnR2xcvno
---
 kalshi-dash/scripts/reconcile-canary.mjs | 25 +++++++++++++++++++++++--
 1 file changed, 23 insertions(+), 2 deletions(-)

diff --git a/kalshi-dash/scripts/reconcile-canary.mjs b/kalshi-dash/scripts/reconcile-canary.mjs
index 945c1f6..93a8ad9 100644
--- a/kalshi-dash/scripts/reconcile-canary.mjs
+++ b/kalshi-dash/scripts/reconcile-canary.mjs
@@ -14,8 +14,29 @@ const url = process.env.KEN_DATABASE_URL;
 if (!url) { console.error('reconcile-canary: KEN_DATABASE_URL not set'); process.exit(2); }
 
 const pool = new pg.Pool({ connectionString: url });
+
+// TK-11999: local Postgres restarts (brew/launchd) made this canary exit 2 when a tick landed
+// mid-restart ("database system is starting up", socket ENOENT, ECONNREFUSED with an empty
+// message). Retry ONLY those transient connection states with backoff; a persistent failure
+// still exits 2 so a genuinely dead DB stays visible.
+const TRANSIENT = new Set(['57P03', '57P01', '57P02', 'ECONNREFUSED', 'ENOENT', 'ECONNRESET']);
+const isTransient = (e) => TRANSIENT.has(e?.code) ||
+  (Array.isArray(e?.errors) && e.errors.some(x => TRANSIENT.has(x?.code))) ||
+  /starting up|shutting down/i.test(e?.message || '');
+async function queryWithRetry(sql, attempts = 6) {
+  for (let i = 1; ; i++) {
+    try { return await pool.query(sql); }
+    catch (e) {
+      if (i >= attempts || !isTransient(e)) throw e;
+      const waitMs = Math.min(5000 * i, 20000);
+      console.error(`[reconcile-canary] transient db error (${e.code || 'no-code'}: ${e.message || 'empty'}) — retry ${i}/${attempts - 1} in ${waitMs / 1000}s`);
+      await new Promise(r => setTimeout(r, waitMs));
+    }
+  }
+}
+
 try {
-  const { rows: [r] } = await pool.query(`
+  const { rows: [r] } = await queryWithRetry(`
     SELECT
       (SELECT COALESCE(SUM(total_returned_cents),0) FROM ken_portfolios)                       AS rollup_returned,
       (SELECT COALESCE(SUM(CASE WHEN pnl_cents>0 THEN cost_cents+pnl_cents
@@ -59,7 +80,7 @@ try {
   } catch { /* CNCP down — the log + exit 1 still carry the alert */ }
   process.exit(1);
 } catch (e) {
-  console.error('[reconcile-canary] error:', e.message);
+  console.error('[reconcile-canary] error:', e.code || 'no-code', e.message || String(e));
   process.exit(2);
 } finally {
   await pool.end();

← cc32555 Bump kalshi-dash to 1.0.5 after verified socket migration  ·  back to Ken  ·  (newest)