← back to Homesonspec
fix(workers): health-probe bind failure must not crash the worker (TK-10240 prod incident)
7ff28c2d0a3131a93f673ebbefb524403180b05a · 2026-08-09 12:14:06 -0700 · Steve
2026-08-09 prod deploy crash-looped: healthServer.listen(9976) hit EADDRINUSE
on the shared Kamatera host and — with no 'error' handler — the unhandled
'error' event became an uncaughtException → process abort → pm2 crash-loop.
That is the exact failure mode TK-10240 set out to eliminate, reintroduced via
the auxiliary liveness probe.
Extract startHealthServer()/closeHealthServer() with a non-fatal .on('error')
(log + continue; the worker keeps draining the queue), bind 127.0.0.1 (local
probe, not public), and guard close() against a server that never bound. Adds
health-server.test.ts incl. an EADDRINUSE-is-non-fatal regression test.
Port default stays 9976 but is now survivable if taken; prod must set a
verified-free WORKER_HEALTH_PORT to actually expose the probe (see deploy memo).
Unit 12/12, integration 7/7, tsc clean.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Files touched
A apps/workers/src/health-server.test.tsA apps/workers/src/health-server.tsM apps/workers/src/index.ts
Diff
commit 7ff28c2d0a3131a93f673ebbefb524403180b05a
Author: Steve <steve@designerwallcoverings.com>
Date: Sun Aug 9 12:14:06 2026 -0700
fix(workers): health-probe bind failure must not crash the worker (TK-10240 prod incident)
2026-08-09 prod deploy crash-looped: healthServer.listen(9976) hit EADDRINUSE
on the shared Kamatera host and — with no 'error' handler — the unhandled
'error' event became an uncaughtException → process abort → pm2 crash-loop.
That is the exact failure mode TK-10240 set out to eliminate, reintroduced via
the auxiliary liveness probe.
Extract startHealthServer()/closeHealthServer() with a non-fatal .on('error')
(log + continue; the worker keeps draining the queue), bind 127.0.0.1 (local
probe, not public), and guard close() against a server that never bound. Adds
health-server.test.ts incl. an EADDRINUSE-is-non-fatal regression test.
Port default stays 9976 but is now survivable if taken; prod must set a
verified-free WORKER_HEALTH_PORT to actually expose the probe (see deploy memo).
Unit 12/12, integration 7/7, tsc clean.
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
---
apps/workers/src/health-server.test.ts | 94 ++++++++++++++++++++++++++++++++++
apps/workers/src/health-server.ts | 57 +++++++++++++++++++++
apps/workers/src/index.ts | 20 +++-----
3 files changed, 158 insertions(+), 13 deletions(-)
diff --git a/apps/workers/src/health-server.test.ts b/apps/workers/src/health-server.test.ts
new file mode 100644
index 00000000..862b0e26
--- /dev/null
+++ b/apps/workers/src/health-server.test.ts
@@ -0,0 +1,94 @@
+import { afterEach, describe, expect, it } from "vitest";
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+import { startHealthServer, closeHealthServer } from "./health-server";
+
+/**
+ * Regression coverage for the 2026-08-09 prod incident: the worker crash-looped
+ * because healthServer.listen() had no 'error' handler, so an EADDRINUSE bind
+ * failure on the shared host became an uncaughtException. A health probe is
+ * auxiliary — its bind failure must NEVER crash the worker.
+ */
+
+const opened: http.Server[] = [];
+afterEach(() => {
+ for (const s of opened) {
+ try {
+ if (s.listening) s.close();
+ } catch {
+ /* ignore */
+ }
+ }
+ opened.length = 0;
+});
+
+function listenReady(server: http.Server): Promise<number> {
+ return new Promise((resolve, reject) => {
+ server.once("error", reject);
+ server.once("listening", () => resolve((server.address() as AddressInfo).port));
+ });
+}
+
+function get(port: number): Promise<{ status: number; body: string }> {
+ return new Promise((resolve, reject) => {
+ http
+ .get({ host: "127.0.0.1", port, path: "/health" }, (res) => {
+ let body = "";
+ res.on("data", (c) => (body += c));
+ res.on("end", () => resolve({ status: res.statusCode ?? 0, body }));
+ })
+ .on("error", reject);
+ });
+}
+
+describe("startHealthServer", () => {
+ it("serves 200 + verdict on /health when healthy", async () => {
+ const server = startHealthServer({ port: 0, host: "127.0.0.1", health: () => ({ healthy: true, reason: "ok" }) });
+ opened.push(server);
+ const port = await listenReady(server);
+ const res = await get(port);
+ expect(res.status).toBe(200);
+ expect(JSON.parse(res.body)).toEqual({ healthy: true, reason: "ok" });
+ });
+
+ it("serves 503 when the queue verdict is unhealthy", async () => {
+ const server = startHealthServer({
+ port: 0,
+ host: "127.0.0.1",
+ health: () => ({ healthy: false, reason: "wedged" }),
+ });
+ opened.push(server);
+ const port = await listenReady(server);
+ const res = await get(port);
+ expect(res.status).toBe(503);
+ expect(JSON.parse(res.body).healthy).toBe(false);
+ });
+
+ it("a bind failure (EADDRINUSE) is NON-FATAL — logs and continues, never throws", async () => {
+ // Occupy a port first.
+ const blocker = http.createServer();
+ opened.push(blocker);
+ const takenPort = await new Promise<number>((resolve) =>
+ blocker.listen(0, "127.0.0.1", () => resolve((blocker.address() as AddressInfo).port)),
+ );
+
+ const errors: string[] = [];
+ // Starting the health server on the SAME port must not throw or crash.
+ const server = startHealthServer({
+ port: takenPort,
+ host: "127.0.0.1",
+ health: () => ({ healthy: true, reason: "ok" }),
+ errorLog: (...a) => errors.push(a.map(String).join(" ")),
+ });
+ opened.push(server);
+
+ // The 'error' event is async; give the event loop a tick to deliver it.
+ await new Promise((r) => setTimeout(r, 50));
+
+ expect(server.listening).toBe(false); // never bound (port taken)...
+ expect(errors.some((e) => /failed to bind/.test(e))).toBe(true); // ...but handled, not thrown.
+ // Reaching this line at all proves the process did not crash.
+ // closeHealthServer must be safe on a server that never bound:
+ expect(() => closeHealthServer(server)).not.toThrow();
+ });
+});
diff --git a/apps/workers/src/health-server.ts b/apps/workers/src/health-server.ts
new file mode 100644
index 00000000..868461d8
--- /dev/null
+++ b/apps/workers/src/health-server.ts
@@ -0,0 +1,57 @@
+import http from "node:http";
+
+export interface HealthVerdict {
+ healthy: boolean;
+ reason: string;
+}
+
+/**
+ * Auxiliary liveness probe for the worker.
+ *
+ * CRITICAL (TK-10240 prod incident 2026-08-09): a bind failure here — e.g.
+ * EADDRINUSE when the chosen port is already taken on the shared Kamatera host,
+ * which crash-looped the worker on the first prod deploy — must NEVER crash the
+ * worker. The probe is AUXILIARY; the worker keeps draining the queue. Without a
+ * `.on('error')` handler the emitted 'error' event is unhandled → uncaughtException
+ * → process abort → pm2 crash-loop: the exact failure mode this ticket set out to
+ * eliminate, reintroduced through the probe. So we log and continue instead.
+ *
+ * Binds 127.0.0.1 by default so the probe is local-only (pm2 / monitoring on the
+ * box), never publicly exposed.
+ */
+export function startHealthServer(opts: {
+ port: number;
+ host?: string;
+ health: () => HealthVerdict;
+ log?: (...args: unknown[]) => void;
+ errorLog?: (...args: unknown[]) => void;
+}): http.Server {
+ const host = opts.host ?? "127.0.0.1";
+ const log = opts.log ?? ((...a: unknown[]) => console.log(...a));
+ const errorLog = opts.errorLog ?? ((...a: unknown[]) => console.error(...a));
+
+ const server = http.createServer((req, res) => {
+ if (req.url === "/health") {
+ const verdict = opts.health();
+ res.writeHead(verdict.healthy ? 200 : 503, { "content-type": "application/json" });
+ res.end(JSON.stringify(verdict));
+ return;
+ }
+ res.writeHead(404);
+ res.end();
+ });
+
+ // Non-fatal: a probe that can't bind must not take the worker down with it.
+ server.on("error", (err) => {
+ const code = (err as NodeJS.ErrnoException).code ?? String(err);
+ errorLog(`workers: health endpoint failed to bind ${host}:${opts.port} (${code}); continuing without it`);
+ });
+
+ server.listen(opts.port, host, () => log(`workers: health endpoint on ${host}:${opts.port}/health`));
+ return server;
+}
+
+/** Close a health server only if it actually bound (close() on a non-listening server throws). */
+export function closeHealthServer(server: http.Server): void {
+ if (server.listening) server.close();
+}
diff --git a/apps/workers/src/index.ts b/apps/workers/src/index.ts
index cf81245f..9b6e6fb3 100644
--- a/apps/workers/src/index.ts
+++ b/apps/workers/src/index.ts
@@ -1,4 +1,3 @@
-import http from "node:http";
import {
work,
stopQueue,
@@ -9,6 +8,7 @@ import {
} from "@homesonspec/shared";
import { meridianHomesAdapter } from "@homesonspec/collector-meridian-homes";
import { runPipeline } from "./pipeline";
+import { startHealthServer, closeHealthServer } from "./health-server";
/**
* Queue-backed worker daemon (pg-boss). Stages stay pure functions; this
@@ -31,18 +31,12 @@ async function main() {
// Liveness probe: distinguishes a healthy-idle worker from a wedged-but-online
// one (pg-boss no longer crashes on a DB error, so pm2 would otherwise report
// "online" through an outage). 503 when the queue has errored recently without
- // a canary recovery — a pm2/monitoring health-check can act on it.
- const healthServer = http.createServer((req, res) => {
- if (req.url === "/health") {
- const verdict = computeQueueHealth(getQueueHealthState(), Date.now());
- res.writeHead(verdict.healthy ? 200 : 503, { "content-type": "application/json" });
- res.end(JSON.stringify(verdict));
- return;
- }
- res.writeHead(404);
- res.end();
+ // a canary recovery — a pm2/monitoring health-check can act on it. Bind failure
+ // (e.g. EADDRINUSE on the shared host) is non-fatal — see startHealthServer.
+ const healthServer = startHealthServer({
+ port: HEALTH_PORT,
+ health: () => computeQueueHealth(getQueueHealthState(), Date.now()),
});
- healthServer.listen(HEALTH_PORT, () => console.log(`workers: health endpoint on :${HEALTH_PORT}/health`));
const canary = startQueueHealthCanary({ queueName: JOB_NAME });
@@ -51,7 +45,7 @@ async function main() {
installGracefulShutdown({
stop: async () => {
clearInterval(canary);
- healthServer.close();
+ closeHealthServer(healthServer);
await stopQueue();
},
});
← 813a5a19 fix: scope Ashton Woods Widen sweep to LD blocks only, fix s
·
back to Homesonspec
·
chore(workers): pin WORKER_HEALTH_PORT=9966 (9976 taken on K ac1caf5d →