`onboarding/sangetsu-lilycolor/scripts/local-recover.sh`

60 lines
#!/bin/bash
# Re-enrich the 1828 error SKUs from the LOCAL image copy (~/dw-lily-images-local),
# not the flaky Henry USB mount — so a mid-run USB unmount can't fail us again.
# Seeded off the 806 good rows, remerges (last-wins) into final. Local, $0, reversible.
set -u
PROJ="$HOME/Projects/Designer-Wallcoverings/onboarding/sangetsu-lilycolor"; cd "$PROJ" || exit 1
ST="$PROJ/staging"
LOCAL_IMG="$HOME/dw-lily-images-local"
GOOD="$ST/enrichment-full-063026A.good-seed.jsonl"
REC="$ST/enrichment-full-063026A.local-recover.jsonl"
FINAL="$ST/enrichment-full-063026A.final.jsonl"
log(){ echo "[localrec $(date '+%H:%M:%S')] $*" >> "$ST/enrich-localrec.log"; }

[ -d "$LOCAL_IMG" ] || { log "FATAL local image dir missing: $LOCAL_IMG"; exit 1; }
[ -f "$GOOD" ] || { log "FATAL good-seed missing: $GOOD"; exit 1; }
log "start. local images=$(ls -1 "$LOCAL_IMG" | wc -l | tr -d ' ') good-seed=$(wc -l < "$GOOD" | tr -d ' ')"

# Enrich from LOCAL dir only (point Sangetsu dir at a nonexistent path so only lily is done).
ENRICH_HENRY_DIR="$LOCAL_IMG" \
ENRICH_SANGETSU_DIR="/nonexistent-sangetsu" \
ENRICH_SHARD_COUNT=1 \
ENRICH_OLLAMA_URL="${ENRICH_OLLAMA_URL:-http://127.0.0.1:11434}" \
ENRICH_DONE_SEEDS="$GOOD" \
python3 -u scripts/enrich-full.py "$REC" >> "$ST/enrich-localrec.log" 2>&1
REC_ROWS=$(wc -l < "$REC" 2>/dev/null | tr -d ' '); REC_ROWS=${REC_ROWS:-0}
log "enrich pass done. recover rows=$REC_ROWS"

# COMPLETENESS GUARD: only remerge->final + touch DONE if the driver actually
# processed (near) all of todo. A driver that dies early (e.g. OOM under swap
# pressure) must NOT overwrite final with a truncated set or falsely signal DONE.
EXPECT=$(( $(ls -1 "$LOCAL_IMG" | wc -l | tr -d ' ') - $(wc -l < "$GOOD" | tr -d ' ') ))
MIN_OK=$(( EXPECT * 98 / 100 ))   # allow ~2% genuine unusable/skip
if [ "$REC_ROWS" -lt "$MIN_OK" ]; then
  log "INCOMPLETE: recover=$REC_ROWS < required=$MIN_OK (todo=$EXPECT). NOT remerging, NOT signaling DONE. Re-run to resume (load_done skips finished)."
  exit 2
fi

# Remerge good + recover (last wins) -> final
python3 - "$GOOD" "$REC" "$FINAL" <<'PY' 2>> "$ST/enrich-localrec.log"
import json,sys
srcs,final=sys.argv[1:-1],sys.argv[-1]
by={}
for p in srcs:
    try:
        for line in open(p):
            line=line.strip()
            if not line: continue
            try: o=json.loads(line)
            except: continue
            s=o.get("sku")
            if s: by[s]=o
    except FileNotFoundError: pass
ok=sum(1 for o in by.values() if not o.get("error") and o.get("usable") is not False)
with open(final,"w") as f:
    for s in sorted(by): f.write(json.dumps(by[s])+"\n")
print(f"[remerge] final unique={len(by)} ok={ok} err={len(by)-ok}")
PY
log "REMERGE done. final=$(wc -l < "$FINAL" | tr -d ' ') rows"
touch "$ST/LOCAL-RECOVER-DONE"; log "ALL DONE"