← back to Doing Viewer
TK-11446: make doing-viewer publish resilient to transient timeouts (retry once, bump local fetch to 30s)
9033cbe8f5a9c6995bcd8c56f142874b11e10eee · 2026-09-12 23:31:59 -0700 · Steve Abrams
Root cause: getLocal() fetch of the local board occasionally exceeded its 20s
timeout while computing summaries for ~56 doing tickets, throwing 'timeout' and
crashing an otherwise-healthy 180s run — cron-fire-canary flapped warn/recovered.
Now retries once with 5s backoff before exiting non-zero; a sustained failure
still exits 1 so the canary genuinely warns. Verified: success->exit0, dead
endpoint->retry->exit1.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01M7Y6Sqsc2PzELRH1a3g4oy
Files touched
Diff
commit 9033cbe8f5a9c6995bcd8c56f142874b11e10eee
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sat Sep 12 23:31:59 2026 -0700
TK-11446: make doing-viewer publish resilient to transient timeouts (retry once, bump local fetch to 30s)
Root cause: getLocal() fetch of the local board occasionally exceeded its 20s
timeout while computing summaries for ~56 doing tickets, throwing 'timeout' and
crashing an otherwise-healthy 180s run — cron-fire-canary flapped warn/recovered.
Now retries once with 5s backoff before exiting non-zero; a sustained failure
still exits 1 so the canary genuinely warns. Verified: success->exit0, dead
endpoint->retry->exit1.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01M7Y6Sqsc2PzELRH1a3g4oy
---
publish-snapshot.js | 29 ++++++++++++++++++++++++++---
1 file changed, 26 insertions(+), 3 deletions(-)
diff --git a/publish-snapshot.js b/publish-snapshot.js
index 36e402f..ce382d0 100644
--- a/publish-snapshot.js
+++ b/publish-snapshot.js
@@ -19,14 +19,19 @@ const REMOTE = 'root@45.61.58.125:/root/Projects/doing-viewer/data/snapshot.json
function getLocal() {
return new Promise((resolve, reject) => {
const auth = 'Basic ' + Buffer.from(`${USER}:${PASS}`).toString('base64');
- http.get(LOCAL, { headers: { Authorization: auth }, timeout: 20000 }, (res) => {
+ // 30s (was 20s): the local server computes plain-word summaries for ~56 doing
+ // tickets on the fly, which under load occasionally crept past 20s and threw
+ // 'timeout', crashing an otherwise-healthy run (TK-11446 flapping).
+ http.get(LOCAL, { headers: { Authorization: auth }, timeout: 30000 }, (res) => {
let d = ''; res.on('data', c => d += c);
res.on('end', () => { try { resolve(JSON.parse(d)); } catch (e) { reject(e); } });
}).on('error', reject).on('timeout', function () { this.destroy(); reject(new Error('timeout')); });
});
}
-(async () => {
+const sleep = ms => new Promise(r => setTimeout(r, ms));
+
+async function publishOnce() {
const data = await getLocal();
const snap = {
syncedAt: new Date().toISOString(),
@@ -43,4 +48,22 @@ function getLocal() {
execFileSync('rsync', ['-az', '--timeout=25', SNAP, REMOTE], { stdio: 'inherit' });
const summarized = snap.items.filter(i => i.eli12).length;
console.log(`[${snap.syncedAt}] published ${snap.count} doing (${summarized} summarized) -> Kamatera`);
-})().catch(e => { console.error('publish failed:', e.message); process.exit(1); });
+}
+
+// Retry ONCE on a transient failure (slow local fetch or a slow Kamatera rsync)
+// before exiting non-zero. A single blip no longer crash-flags the job; a
+// SUSTAINED failure still exits 1 so cron-fire-canary genuinely warns (TK-11446).
+(async () => {
+ try {
+ await publishOnce();
+ } catch (e1) {
+ console.error(`publish attempt 1 failed: ${e1.message} — retrying once in 5s`);
+ await sleep(5000);
+ try {
+ await publishOnce();
+ } catch (e2) {
+ console.error(`publish failed (after retry): ${e2.message}`);
+ process.exit(1);
+ }
+ }
+})();
← 311bd4d creds-safe fetch guard: resolve relative fetch vs credential
·
back to Doing Viewer
·
TK-11446: exit 0 (not 1) when doing-viewer source :9790 is t 3fad9e6 →