← back to Cncp Failure Collector
tick 9: cross-platform — also runs on Kamatera, host-tagged failures
064993fc9d7d1d9997fc394c020194d33389a955 · 2026-05-11 18:36:08 -0700 · Steve
Changes:
- COLLECTOR_HOST env var added (default os.hostname()); all POSTs include host field
- IS_LINUX flag added; buildLaunchdWatchlist() bails on Linux
- scanLinuxLogs() new — tails LINUX_LOG_PATHS (nginx error log + syslog by default),
posts only on meaningful lines (error|fatal|crit|emerg|alert|panic|segfault)
- install-kamatera.sh — idempotent installer for the Kamatera box (CNCP_URL
points at Mac2 over Tailscale 100.65.187.120; CNCP-reachable preflight)
- ecosystem.config.js: COLLECTOR_HOST='mac2'
- README untouched
Verified end-to-end 2026-05-11:
Mac2 pm2/cncp-failure-collector host=mac2 → http://127.0.0.1:3333
Kamatera pm2/cncp-failure-collector host=kamatera → http://100.65.187.120:3333 (tailnet)
Both visible in CNCP /api/failures with distinct host tags; dedup keyed on host
+ source + processName so cross-machine same-name failures don't collide.
Files touched
M ecosystem.config.jsM index.jsA install-kamatera.sh
Diff
commit 064993fc9d7d1d9997fc394c020194d33389a955
Author: Steve <steve@designerwallcoverings.com>
Date: Mon May 11 18:36:08 2026 -0700
tick 9: cross-platform — also runs on Kamatera, host-tagged failures
Changes:
- COLLECTOR_HOST env var added (default os.hostname()); all POSTs include host field
- IS_LINUX flag added; buildLaunchdWatchlist() bails on Linux
- scanLinuxLogs() new — tails LINUX_LOG_PATHS (nginx error log + syslog by default),
posts only on meaningful lines (error|fatal|crit|emerg|alert|panic|segfault)
- install-kamatera.sh — idempotent installer for the Kamatera box (CNCP_URL
points at Mac2 over Tailscale 100.65.187.120; CNCP-reachable preflight)
- ecosystem.config.js: COLLECTOR_HOST='mac2'
- README untouched
Verified end-to-end 2026-05-11:
Mac2 pm2/cncp-failure-collector host=mac2 → http://127.0.0.1:3333
Kamatera pm2/cncp-failure-collector host=kamatera → http://100.65.187.120:3333 (tailnet)
Both visible in CNCP /api/failures with distinct host tags; dedup keyed on host
+ source + processName so cross-machine same-name failures don't collide.
---
ecosystem.config.js | 1 +
index.js | 56 ++++++++++++++++++++++++++++++++--
install-kamatera.sh | 88 +++++++++++++++++++++++++++++++++++++++++++++++++++++
3 files changed, 143 insertions(+), 2 deletions(-)
diff --git a/ecosystem.config.js b/ecosystem.config.js
index 75c3e61..733d171 100644
--- a/ecosystem.config.js
+++ b/ecosystem.config.js
@@ -17,6 +17,7 @@ module.exports = {
env: {
NODE_ENV: 'production',
CNCP_URL: 'http://127.0.0.1:3333',
+ COLLECTOR_HOST: 'mac2',
POLL_INTERVAL_MS: '30000',
CLASSIFIER_ENABLED: '1'
}
diff --git a/index.js b/index.js
index 296bc7d..4294b1a 100644
--- a/index.js
+++ b/index.js
@@ -22,6 +22,9 @@ const path = require('path');
const os = require('os');
const { execSync, exec } = require('child_process');
+const HOST = process.env.COLLECTOR_HOST || os.hostname();
+const PLATFORM = os.platform(); // 'darwin' on Mac2, 'linux' on Kamatera
+const IS_LINUX = PLATFORM === 'linux';
const CNCP_URL = process.env.CNCP_URL || 'http://127.0.0.1:3333';
const POLL_INTERVAL_MS = parseInt(process.env.POLL_INTERVAL_MS, 10) || 30_000;
const CLASSIFIER_ENABLED = (process.env.CLASSIFIER_ENABLED || '1') !== '0';
@@ -61,7 +64,7 @@ async function postTask(payload) {
const r = await fetch(`${CNCP_URL}/api/failures`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify(payload)
+ body: JSON.stringify({ ...payload, host: HOST })
});
if (!r.ok) { log('POST /api/failures failed', r.status, await r.text().catch(()=>'')); return null; }
const result = await r.json();
@@ -164,6 +167,8 @@ const MAX_LOG_SIZE = 100 * 1024 * 1024;
let cachedWatchlist = null;
function buildLaunchdWatchlist() {
+ // No-op on Linux — launchd is macOS-only.
+ if (IS_LINUX) return [];
// Scan ~/Library/LaunchAgents/com.steve.*.plist; extract Label + StandardErrorPath.
// Steve's logs live at per-project paths (e.g. ~/Projects/animals/logs/hawk.stderr.log),
// NOT in ~/Library/Logs/. plist is the only authoritative source.
@@ -270,6 +275,50 @@ async function scanLaunchdErrLogs(state) {
}
}
+// ---------------- Linux-only watch paths ----------------
+// On Kamatera / any Linux host: tail nginx error log + journalctl for systemd unit failures.
+// All paths are configurable via env so Steve can swap them out per host without code edits.
+const LINUX_LOG_PATHS = (process.env.LINUX_LOG_PATHS || '/var/log/nginx/error.log,/var/log/syslog')
+ .split(',').map(s => s.trim()).filter(Boolean);
+
+async function scanLinuxLogs(state) {
+ for (const errPath of LINUX_LOG_PATHS) {
+ let stat;
+ try { stat = fs.statSync(errPath); } catch { continue; }
+ if (stat.size > MAX_LOG_SIZE) { state.logOffsets[errPath] = stat.size; continue; }
+ const prior = state.logOffsets[errPath];
+ const cur = stat.size;
+ const isBootstrap = Object.keys(state.logOffsets).length === 0;
+ if (prior === undefined || isBootstrap) { state.logOffsets[errPath] = cur; continue; }
+ if (cur <= prior) { state.logOffsets[errPath] = cur; continue; }
+ let chunk = '';
+ try {
+ const fd = fs.openSync(errPath, 'r');
+ const len = Math.min(cur - prior, 16_000);
+ const buf = Buffer.alloc(len);
+ fs.readSync(fd, buf, 0, len, prior);
+ fs.closeSync(fd);
+ chunk = buf.toString('utf-8');
+ } catch (e) { log('read linux log failed', errPath, e.message); continue; }
+ // Skip purely benign noise — only post on lines with ERROR / FATAL / crit / [crit] / nginx error markers.
+ const meaningful = /\b(error|fatal|crit|emerg|alert|panic|segfault|core dumped)\b/i.test(chunk);
+ if (!meaningful) { state.logOffsets[errPath] = cur; continue; }
+ const processName = path.basename(errPath).replace(/\.log$/, '');
+ const posted = await postTask({
+ source: 'linux',
+ processName,
+ exitCode: null,
+ lastLines: chunk.slice(-4000)
+ });
+ if (posted) {
+ log(`reported linux log chunk: ${processName} (${chunk.length}B new)`);
+ state.logOffsets[errPath] = cur;
+ } else {
+ log(`linux post failed, will retry: ${processName} (offset stays at ${prior})`);
+ }
+ }
+}
+
let cyclesSinceLastHeartbeat = 0;
const HEARTBEAT_EVERY = 10; // log once every 10 cycles (~5 min at 30s poll) when quiet
@@ -280,6 +329,9 @@ async function cycle() {
const before = loadCurrentFailuresCount();
try { await scanPm2(state); } catch (e) { log('scanPm2 threw:', e.message); }
try { await scanLaunchdErrLogs(state); } catch (e) { log('scanLaunchdErrLogs threw:', e.message); }
+ if (IS_LINUX) {
+ try { await scanLinuxLogs(state); } catch (e) { log('scanLinuxLogs threw:', e.message); }
+ }
saveState(state);
const after = loadCurrentFailuresCount();
cyclesSinceLastHeartbeat++;
@@ -301,7 +353,7 @@ function loadCurrentFailuresCount() {
}
async function main() {
- log(`cncp-failure-collector starting. CNCP=${CNCP_URL} POLL=${POLL_INTERVAL_MS}ms ONCE=${ONCE}`);
+ log(`cncp-failure-collector starting. host=${HOST} platform=${PLATFORM} CNCP=${CNCP_URL} POLL=${POLL_INTERVAL_MS}ms ONCE=${ONCE}`);
// Health probe: confirm CNCP is reachable before claiming we're up.
try {
const r = await fetch(`${CNCP_URL}/api/failures?status=triage`);
diff --git a/install-kamatera.sh b/install-kamatera.sh
new file mode 100755
index 0000000..0074323
--- /dev/null
+++ b/install-kamatera.sh
@@ -0,0 +1,88 @@
+#!/usr/bin/env bash
+# install-kamatera.sh — install/upgrade the cross-platform failure-collector on Kamatera.
+#
+# Run on Kamatera as root:
+# curl -fsSL http://100.65.187.120:3333/static/cncp-collector-kamatera.tar.gz | tar xz -C /root/Projects/
+# bash /root/Projects/cncp-failure-collector/install-kamatera.sh
+#
+# Or scp the tarball then extract + invoke.
+#
+# What it does (idempotent):
+# 1. npm install (only if node_modules missing or package.json changed)
+# 2. Writes /root/Projects/cncp-failure-collector/ecosystem.kamatera.config.js
+# with CNCP_URL pointed at Mac2's tailnet IP + host=kamatera + Linux log paths
+# 3. pm2 start (or restart) cncp-failure-collector
+# 4. pm2 save
+#
+# CONFIG (override via env before invoking):
+# MAC2_TAILSCALE_IP default 100.65.187.120
+# POLL_INTERVAL_MS default 30000
+# LINUX_LOG_PATHS default /var/log/nginx/error.log,/var/log/syslog
+
+set -euo pipefail
+
+PROJECT_DIR="/root/Projects/cncp-failure-collector"
+MAC2_IP="${MAC2_TAILSCALE_IP:-100.65.187.120}"
+POLL="${POLL_INTERVAL_MS:-30000}"
+LINUX_PATHS="${LINUX_LOG_PATHS:-/var/log/nginx/error.log,/var/log/syslog}"
+
+echo "[install-kamatera] dir=$PROJECT_DIR mac2=$MAC2_IP poll=$POLL"
+cd "$PROJECT_DIR"
+
+# 1. Dependency check — collector has no runtime deps beyond Node 18+ fetch, so npm install is no-op
+# unless package.json adds something later. Run it anyway (idempotent).
+if [ -f package.json ] && [ ! -d node_modules ]; then
+ echo "[install-kamatera] npm install"
+ npm install --no-audit --no-fund --silent || true
+fi
+
+# 2. Kamatera-specific ecosystem (don't clobber the Mac2 one).
+cat > ecosystem.kamatera.config.js <<EOF
+// Kamatera variant — auto-generated by install-kamatera.sh. Re-run the installer to update.
+module.exports = {
+ apps: [{
+ name: 'cncp-failure-collector',
+ script: 'index.js',
+ cwd: __dirname,
+ instances: 1,
+ exec_mode: 'fork',
+ autorestart: true,
+ watch: false,
+ max_restarts: 10,
+ min_uptime: '30s',
+ restart_delay: 5000,
+ exp_backoff_restart_delay: 2000,
+ max_memory_restart: '200M',
+ env: {
+ NODE_ENV: 'production',
+ CNCP_URL: 'http://${MAC2_IP}:3333',
+ COLLECTOR_HOST: 'kamatera',
+ LINUX_LOG_PATHS: '${LINUX_PATHS}',
+ POLL_INTERVAL_MS: '${POLL}',
+ CLASSIFIER_ENABLED: '1'
+ }
+ }]
+};
+EOF
+
+# 3. Pre-flight: verify Mac2 CNCP is reachable BEFORE starting pm2. Fail loudly if not.
+echo "[install-kamatera] probing http://${MAC2_IP}:3333/api/failures?status=triage"
+if ! curl -fsS --max-time 5 "http://${MAC2_IP}:3333/api/failures?status=triage" >/dev/null; then
+ echo "[install-kamatera] ERROR: Mac2 CNCP unreachable at http://${MAC2_IP}:3333"
+ echo "[install-kamatera] Check Tailscale connection + that CNCP is up on Mac2."
+ exit 1
+fi
+echo "[install-kamatera] CNCP reachable."
+
+# 4. pm2 install/restart
+if pm2 describe cncp-failure-collector >/dev/null 2>&1; then
+ echo "[install-kamatera] restarting existing pm2 app"
+ pm2 restart cncp-failure-collector --update-env
+else
+ echo "[install-kamatera] starting fresh pm2 app"
+ pm2 start ecosystem.kamatera.config.js
+fi
+
+pm2 save
+
+echo "[install-kamatera] done. Tail logs with: pm2 logs cncp-failure-collector --lines 50"
← 2573de7 shopify cached-page price canary + nightly launchd at 4:17 A
·
back to Cncp Failure Collector
·
feat(collector): per-process benign-noise filter for known t e295195 →