[object Object]

← back to Designer Wallcoverings

sangetsu: add swap-guarded per-pattern enrichment runner + autonomous download->enrich pipeline (session close)

48b151f81fb121227554f0012147383ca679c9c2 · 2026-07-02 08:03:54 -0700 · Steve

Files touched

Diff

commit 48b151f81fb121227554f0012147383ca679c9c2
Author: Steve <steve@designerwallcoverings.com>
Date:   Thu Jul 2 08:03:54 2026 -0700

    sangetsu: add swap-guarded per-pattern enrichment runner + autonomous download->enrich pipeline (session close)
---
 .../scripts/sangetsu-enrich-guarded.sh             | 44 ++++++++++++++++++++++
 .../scripts/sangetsu-pipeline.sh                   | 16 ++++++++
 2 files changed, 60 insertions(+)

diff --git a/onboarding/sangetsu-lilycolor/scripts/sangetsu-enrich-guarded.sh b/onboarding/sangetsu-lilycolor/scripts/sangetsu-enrich-guarded.sh
new file mode 100755
index 00000000..f7c216c5
--- /dev/null
+++ b/onboarding/sangetsu-lilycolor/scripts/sangetsu-enrich-guarded.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+# Per-PATTERN Sangetsu enrichment (styles/color-hex/material/patterns/imageType)
+# on the local representative-image cache — same swap-guarded, memory-safe pacing
+# as the Lilycolor runner so it never forces inference under pressure.
+# 623 rep images (one per pattern) → ~4h. Pillow hex + ollama vision, $0 local.
+# Resumable (seeds off rows already in the output). Reversible (JSONL only).
+set -u
+PROJ="$HOME/Projects/Designer-Wallcoverings/onboarding/sangetsu-lilycolor"; cd "$PROJ" || exit 1
+ST="$PROJ/staging"
+CACHE="$HOME/dw-sangetsu-images-local"
+OUT="$ST/sangetsu-enrichment.jsonl"
+CONT_ABOVE_MB=1800   # box has headroom now; resume when swap free > this
+STOP_BELOW_MB=800     # pause when swap free < this
+log(){ echo "[sg-guard $(date '+%H:%M:%S')] $*" >> "$ST/sangetsu-enrich-guard.log"; }
+swap_free_mb(){ sysctl -n vm.swapusage 2>/dev/null | sed -E 's/.*free = ([0-9.]+)M.*/\1/' | cut -d. -f1; }
+
+[ -d "$CACHE" ] || { log "FATAL cache missing $CACHE"; exit 1; }
+log "start. cache=$(ls -1 "$CACHE"/*.jpg 2>/dev/null | wc -l | tr -d ' ') imgs done=$(wc -l < "$OUT" 2>/dev/null | tr -d ' ')"
+
+# wait for headroom before launching
+while true; do
+  fm=$(swap_free_mb); fm=${fm:-0}
+  [ "$fm" -gt "$CONT_ABOVE_MB" ] && break
+  log "waiting for headroom (swap free ${fm}M < ${CONT_ABOVE_MB}M)"; sleep 30
+done
+log "headroom OK (swap $(swap_free_mb)M) — launching"
+
+ENRICH_HENRY_DIR="$CACHE" ENRICH_SANGETSU_DIR="/nonexistent" ENRICH_SHARD_COUNT=1 \
+ENRICH_OLLAMA_URL=http://127.0.0.1:11434 ENRICH_VL_MAX=768 ENRICH_DONE_SEEDS="$OUT" \
+python3 -u scripts/enrich-full.py "$OUT" >> "$ST/sangetsu-enrich-guard.log" 2>&1 &
+DPID=$!
+log "driver PID $DPID launched (swap-paced)"
+
+paused=0
+while kill -0 "$DPID" 2>/dev/null; do
+  fm=$(swap_free_mb); fm=${fm:-0}
+  if [ "$paused" -eq 0 ] && [ "$fm" -lt "$STOP_BELOW_MB" ]; then
+    kill -STOP "$DPID" 2>/dev/null; paused=1; log "PAUSE (swap ${fm}M)"
+  elif [ "$paused" -eq 1 ] && [ "$fm" -gt "$CONT_ABOVE_MB" ]; then
+    kill -CONT "$DPID" 2>/dev/null; paused=0; log "RESUME (swap ${fm}M)"
+  fi
+  sleep 5
+done
+log "driver exited. rows=$(wc -l < "$OUT" 2>/dev/null | tr -d ' ')"; touch "$ST/SANGETSU-ENRICH-DONE"; log "DONE"
diff --git a/onboarding/sangetsu-lilycolor/scripts/sangetsu-pipeline.sh b/onboarding/sangetsu-lilycolor/scripts/sangetsu-pipeline.sh
new file mode 100755
index 00000000..2171b460
--- /dev/null
+++ b/onboarding/sangetsu-lilycolor/scripts/sangetsu-pipeline.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+# Autonomous chain: wait for the rep-image download to finish, then run the
+# swap-guarded per-pattern Sangetsu enrichment. Self-completing.
+set -u
+PROJ="$HOME/Projects/Designer-Wallcoverings/onboarding/sangetsu-lilycolor"; cd "$PROJ"
+ST="$PROJ/staging"; CACHE="$HOME/dw-sangetsu-images-local"
+log(){ echo "[sg-pipe $(date '+%H:%M:%S')] $*" >> "$ST/sangetsu-pipeline.log"; }
+# wait until the download python is gone AND cache size stabilizes
+prev=-1
+while pgrep -f 'sangetsu-staging.jsonl.*urllib\|urllib.*sangetsu' >/dev/null 2>&1 || true; do
+  n=$(ls -1 "$CACHE"/*.jpg 2>/dev/null | wc -l | tr -d ' ')
+  if [ "$n" = "$prev" ]; then log "cache stable at $n imgs — download done"; break; fi
+  prev=$n; log "waiting on download (cache=$n)"; sleep 45
+done
+log "launching enrichment on $(ls -1 "$CACHE"/*.jpg 2>/dev/null | wc -l | tr -d ' ') imgs"
+exec bash "$PROJ/scripts/sangetsu-enrich-guarded.sh"

← 5e8d6afa sangetsu: case-insensitive banned-word guard in overview ext  ·  back to Designer Wallcoverings  ·  auto-save: 2026-07-02T08:16:24 (4 files) — DW-Programming/Im b3ca297d →