`onboarding/sangetsu-lilycolor/scripts/retry-errors.sh`

61 lines
#!/bin/bash
# Waits for the finisher (ENRICH-DONE), then retries any SKU that landed as an
# error/usable:false row during the Mac2 ollama blip — by re-running enrich seeded
# off SUCCESSFUL rows only (so error SKUs are NOT counted done and get redone).
# Re-merges (last-wins) into the final. Local, $0, reversible (JSONL only).
set -u
PROJ="$HOME/Projects/Designer-Wallcoverings/onboarding/sangetsu-lilycolor"; cd "$PROJ" || exit 1
ST="$PROJ/staging"; FINAL="$ST/enrichment-full-063026A.final.jsonl"
GOOD="$ST/enrichment-full-063026A.good-seed.jsonl"; RETRY="$ST/enrichment-full-063026A.retry.jsonl"
log(){ echo "[retry $(date '+%H:%M:%S')] $*" >> "$ST/enrich-retry.log"; }

log "waiting for ENRICH-DONE"
while [ ! -f "$ST/ENRICH-DONE" ]; do sleep 120; done
log "finisher done. scanning final for error/unusable rows"

# good-seed = final rows WITHOUT an error field and not usable:false
python3 - "$FINAL" "$GOOD" <<'PY' 2>> "$ST/enrich-retry.log"
import json,sys
src,good=sys.argv[1],sys.argv[2]
n_bad=0
with open(good,"w") as g:
    for line in open(src):
        line=line.strip()
        if not line: continue
        try: o=json.loads(line)
        except: continue
        if o.get("error") or o.get("usable") is False:
            n_bad+=1; continue
        g.write(json.dumps(o)+"\n")
print(f"[scan] good rows kept, {n_bad} error/unusable to retry")
PY

BAD=$(python3 -c "import json;print(sum(1 for l in open('$FINAL') if l.strip() and (json.loads(l).get('error') or json.loads(l).get('usable') is False)))" 2>/dev/null || echo 0)
if [ "${BAD:-0}" -eq 0 ]; then log "no error rows — nothing to retry"; touch "$ST/RETRY-DONE"; exit 0; fi
log "retrying $BAD SKUs (seeded off good rows only)"

ENRICH_SHARD_COUNT=1 ENRICH_OLLAMA_URL=http://127.0.0.1:11434 \
ENRICH_DONE_SEEDS="$GOOD" python3 -u scripts/enrich-full.py "$RETRY" >> "$ST/enrich-retry.log" 2>&1
log "retry run done. retry rows=$(wc -l < "$RETRY" 2>/dev/null)"

# merge good + retry (last wins) -> final
python3 - "$GOOD" "$RETRY" "$FINAL" <<'PY' 2>> "$ST/enrich-retry.log"
import json,sys
srcs,final=sys.argv[1:-1],sys.argv[-1]
by={}
for p in srcs:
    try:
        for line in open(p):
            line=line.strip()
            if not line: continue
            try:o=json.loads(line)
            except:continue
            s=o.get("sku")
            if s: by[s]=o
    except FileNotFoundError: pass
with open(final,"w") as f:
    for s in sorted(by): f.write(json.dumps(by[s])+"\n")
print(f"[remerge] final unique SKUs = {len(by)}")
PY
log "REMERGE done. final=$(wc -l < "$FINAL") rows"; touch "$ST/RETRY-DONE"; log "ALL DONE"