[object Object]

← back to Homesonspec

ops: nightly builder-import sweep (launchd com.steve.homesonspec-imports)

4ad39125091b0e07a3cd9c35ec8c56f17edba8df · 2026-07-30 10:07:38 -0700 · Steve

Auto-refreshes every active, non-synthetic SourceRegistry adapter (28 builders
incl. all publicly-traded ones) into the local catalog nightly at 3:15am.
Driven off a live registry query (new activations auto-included), single-flight
locked, per-adapter 30min timeout with cheap snapshot-resume, idempotent upserts.

Author: steve@designerwallcoverings.com

Files touched

Diff

commit 4ad39125091b0e07a3cd9c35ec8c56f17edba8df
Author: Steve <steve@designerwallcoverings.com>
Date:   Thu Jul 30 10:07:38 2026 -0700

    ops: nightly builder-import sweep (launchd com.steve.homesonspec-imports)
    
    Auto-refreshes every active, non-synthetic SourceRegistry adapter (28 builders
    incl. all publicly-traded ones) into the local catalog nightly at 3:15am.
    Driven off a live registry query (new activations auto-included), single-flight
    locked, per-adapter 30min timeout with cheap snapshot-resume, idempotent upserts.
    
    Author: steve@designerwallcoverings.com
---
 .gitignore                              |  3 ++
 ops/com.steve.homesonspec-imports.plist | 28 +++++++++++++++
 ops/run-all-imports.sh                  | 62 +++++++++++++++++++++++++++++++++
 3 files changed, 93 insertions(+)

diff --git a/.gitignore b/.gitignore
index b257910e..13e743b2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -26,3 +26,6 @@ screenrecord/
 
 # rollback file is regenerated at cutover time; never commit (holds live creds)
 ROLLBACK-DNS.txt
+ops/import-sweep.log
+ops/import-sweep.launchd.log
+ops/.import-sweep.lock
diff --git a/ops/com.steve.homesonspec-imports.plist b/ops/com.steve.homesonspec-imports.plist
new file mode 100644
index 00000000..b9faaa25
--- /dev/null
+++ b/ops/com.steve.homesonspec-imports.plist
@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>com.steve.homesonspec-imports</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>/bin/bash</string>
+    <string>/Users/macstudio3/Projects/homesonspec/ops/run-all-imports.sh</string>
+  </array>
+  <key>StartCalendarInterval</key>
+  <dict>
+    <key>Hour</key>
+    <integer>3</integer>
+    <key>Minute</key>
+    <integer>15</integer>
+  </dict>
+  <key>RunAtLoad</key>
+  <false/>
+  <key>StandardOutPath</key>
+  <string>/Users/macstudio3/Projects/homesonspec/ops/import-sweep.launchd.log</string>
+  <key>StandardErrorPath</key>
+  <string>/Users/macstudio3/Projects/homesonspec/ops/import-sweep.launchd.log</string>
+  <key>ProcessType</key>
+  <string>Background</string>
+</dict>
+</plist>
diff --git a/ops/run-all-imports.sh b/ops/run-all-imports.sh
new file mode 100755
index 00000000..92853e17
--- /dev/null
+++ b/ops/run-all-imports.sh
@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+# Nightly builder-import sweep for homesonspec.
+#
+# Runs every ACTIVE, non-synthetic SourceRegistry adapter through the pipeline
+# into the local DB, keeping the whole builder catalog fresh automatically.
+# Idempotent by design: the pipeline upserts by canonicalKey and skips pages
+# whose bytes are unchanged (contentHash dedup), so an unchanged feed is a fast
+# no-op and a re-run never duplicates homes. A slow rate-limited crawl (e.g.
+# khovnanian) that hits its per-adapter timeout resumes cheaply next run.
+#
+# Driven off a live DB query — no hardcoded builder list — so newly-activated
+# sources are picked up automatically. NEVER runs FIXTURE/SYNTHETIC sources
+# (demo inventory must never surface on the real catalog).
+#
+# Wired to launchd: com.steve.homesonspec-imports (nightly). Manual run: just
+# execute this script. Tail: tail -f ~/Projects/homesonspec/ops/import-sweep.log
+set -uo pipefail
+
+REPO="$HOME/Projects/homesonspec"
+LOG="$REPO/ops/import-sweep.log"
+LOCK="$REPO/ops/.import-sweep.lock"
+PER_ADAPTER_TIMEOUT=1800   # 30 min cap per adapter; slow crawls resume next run
+
+# Single-flight: don't stack sweeps if the previous one is still running.
+if [ -e "$LOCK" ] && kill -0 "$(cat "$LOCK" 2>/dev/null)" 2>/dev/null; then
+  echo "$(date '+%F %T') sweep already running (pid $(cat "$LOCK")) — skipping" >> "$LOG"
+  exit 0
+fi
+echo $$ > "$LOCK"
+trap 'rm -f "$LOCK"' EXIT
+
+cd "$REPO" || { echo "repo not found: $REPO" >> "$LOG"; exit 1; }
+set -a; . ./.env; set +a
+
+{
+  echo "=================================================================="
+  echo "=== IMPORT SWEEP START $(date '+%F %T') ==="
+} >> "$LOG"
+
+# Active, real (non-fixture/synthetic) adapters — live from the registry.
+KEYS=$(psql "$DATABASE_URL" -tAc "select key from \"SourceRegistry\" where active = true and \"collectionMethod\" not in ('FIXTURE','SYNTHETIC') order by key;")
+
+if [ -z "$KEYS" ]; then
+  echo "no active adapters found — nothing to sweep" >> "$LOG"
+  exit 0
+fi
+
+cd "$REPO/apps/workers" || exit 1
+ok=0; fail=0
+for k in $KEYS; do
+  echo "--- $k  $(date '+%T') ---" >> "$LOG"
+  if timeout "$PER_ADAPTER_TIMEOUT" npx tsx src/cli.ts --adapter="$k" >> "$LOG" 2>&1; then
+    ok=$((ok+1))
+  else
+    echo "  ⚠ $k failed or timed out (resumes next run)" >> "$LOG"
+    fail=$((fail+1))
+  fi
+done
+
+{
+  echo "=== IMPORT SWEEP DONE $(date '+%F %T') — ok=$ok fail=$fail ==="
+} >> "$LOG"

← 891f7980 auto-save: 2026-07-30T08:46:20 (1 files) — pnpm-lock.yaml  ·  back to Homesonspec  ·  fix: lennar-mac-refresh PATH — add /opt/homebrew/bin so npx 79ed5aa4 →