`onboarding/sangetsu-lilycolor/scripts/enrich-local-supervised.sh`

81 lines
#!/usr/bin/env bash
# enrich-local-supervised.sh — THE single canonical Lily recovery runner.
#
# Combines: (1) supervise.sh's jetsam-survival respawn loop, and (2) the local
# cache + good-seed + remerge logic. Reads images from the STABLE local SSD
# cache (.image-cache/lily), NOT flaky /Volumes/Henry — so image_not_found under
# sustained I/O can't happen. Resumable, $0 local, JSONL-only (reversible).
#
# Completion = a clean enrich pass (rc==0) that produced NO new rows.
set -u
PROJ="$HOME/Projects/Designer-Wallcoverings/onboarding/sangetsu-lilycolor"
cd "$PROJ" || exit 1
ST="$PROJ/staging"; CACHE="$PROJ/.image-cache/lily"
FINAL="$ST/enrichment-full-063026A.final.jsonl"
GOOD="$ST/enrichment-full-063026A.good-seed2.jsonl"
REC="$ST/enrichment-full-063026A.local-recovery.jsonl"
LOG="$ST/enrich-localsup.log"
log(){ echo "[localsup $(date '+%H:%M:%S')] $*" >> "$LOG"; }

[ -d "$CACHE" ] || { log "FATAL cache missing $CACHE"; exit 1; }
NC=$(find "$CACHE" -type f 2>/dev/null | wc -l | tr -d ' ')
log "START local-supervised recovery. cache files=$NC"

# good-seed2 = current final rows WITHOUT error and not usable:false
python3 - "$FINAL" "$GOOD" <<'PY' 2>> "$LOG"
import json,sys
src,good=sys.argv[1],sys.argv[2]; bad=0; ok=0
with open(good,"w") as g:
    for line in open(src):
        line=line.strip()
        if not line: continue
        try:o=json.loads(line)
        except:continue
        if o.get("error") or o.get("usable") is False: bad+=1; continue
        ok+=1; g.write(json.dumps(o)+"\n")
print(f"[seed] good={ok} to-redo={bad}")
PY

count(){ wc -l < "$REC" 2>/dev/null | tr -d ' ' || echo 0; }
# supervise loop: relaunch on any jetsam/non-clean exit; stop on clean+no-new
while true; do
  before="$(count)"
  ENRICH_HENRY_DIR="$CACHE" ENRICH_SHARD_COUNT=1 ENRICH_OLLAMA_URL=http://127.0.0.1:11434 \
    ENRICH_DONE_SEEDS="$GOOD" python3 -u scripts/enrich-full.py "$REC" >> "$LOG" 2>&1
  rc=$?; after="$(count)"
  if [ "$rc" -eq 0 ] && [ "$before" = "$after" ]; then
    log "COMPLETE (clean exit, no new rows at ${after})"; break
  fi
  log "pass rc=$rc rows ${before}->${after} — relaunch in 5s"; sleep 5
done

# Remerge ALL enrichment sources (last-wins) into final so nothing is lost.
python3 - "$FINAL" <<'PY' 2>> "$LOG"
import json,sys,glob,os
ST=os.path.dirname(sys.argv[1]); final=sys.argv[1]
srcs=[final, os.path.join(ST,"enrichment-full-063026A.good-seed2.jsonl"),
      os.path.join(ST,"enrichment-full-063026A.local-recovery.jsonl"),
      os.path.join(ST,"enrichment-full-063026A.local-recover.jsonl"),
      os.path.join(ST,"enrichment-full-063026A.jsonl")]
by={}
def better(new,old):
    # prefer a real enrichment row over an error/unusable one
    def score(o): return 0 if (o.get("error") or o.get("usable") is False) else 1
    return score(new) >= score(old)
for p in srcs:
    if not os.path.exists(p): continue
    for line in open(p):
        line=line.strip()
        if not line: continue
        try:o=json.loads(line)
        except:continue
        s=o.get("sku")
        if not s: continue
        if s not in by or better(o,by[s]): by[s]=o
ok=sum(1 for o in by.values() if not o.get("error") and o.get("usable") is not False)
with open(final,"w") as f:
    for s in sorted(by): f.write(json.dumps(by[s])+"\n")
print(f"[remerge] final={len(by)} ok={ok} err={len(by)-ok}")
PY
log "REMERGE done. final=$(wc -l < "$FINAL") rows"; touch "$ST/LOCALRE-DONE"; log "ALL DONE"