`onboarding/sangetsu-lilycolor/scripts/finish-enrich.sh`

57 lines
#!/bin/bash
# Waits for the two enrichment shards to finish, runs a completeness sweep to
# catch any SKU neither shard covered, then merges all sources into one deduped
# final JSONL. Fully local ($0), reversible (produces JSONL only; no DB/Shopify).
set -u
PROJ="$HOME/Projects/Designer-Wallcoverings/onboarding/sangetsu-lilycolor"
cd "$PROJ" || exit 1
ST="$PROJ/staging"
SEED="$ST/enrichment-full-063026A.jsonl"
S0="$ST/enrichment-full-063026A.shard0.jsonl"
S1="$ST/enrichment-full-063026A.shard1.jsonl"
SWEEP="$ST/enrichment-full-063026A.sweep.jsonl"
FINAL="$ST/enrichment-full-063026A.final.jsonl"

log(){ echo "[finish $(date '+%H:%M:%S')] $*" >> "$ST/enrich-finish.log"; }

log "waiting for shards to exit"
while ps -p "$(cat "$ST/enrich-shard0.pid" 2>/dev/null)" >/dev/null 2>&1 \
   || ps -p "$(cat "$ST/enrich-shard1.pid" 2>/dev/null)" >/dev/null 2>&1; do
  sleep 60
done
log "both shards done. shard0=$(wc -l < "$S0" 2>/dev/null) shard1=$(wc -l < "$S1" 2>/dev/null)"

# Completeness sweep: single local stream, seeded off ALL sources so it only
# does SKUs no shard reached (gaps from any partition drift). Skips everything done.
log "sweep start"
ENRICH_SHARD_COUNT=1 \
ENRICH_OLLAMA_URL=http://127.0.0.1:11434 \
ENRICH_DONE_SEEDS="$SEED:$S0:$S1" \
python3 -u scripts/enrich-full.py "$SWEEP" >> "$ST/enrich-sweep.log" 2>&1
log "sweep done. sweep rows=$(wc -l < "$SWEEP" 2>/dev/null)"

# Merge all sources, dedup by sku (last occurrence wins).
python3 - "$SEED" "$S0" "$S1" "$SWEEP" "$FINAL" <<'PY' 2>> "$ST/enrich-finish.log"
import json, sys
srcs, final = sys.argv[1:-1], sys.argv[-1]
by_sku = {}
for p in srcs:
    try:
        for line in open(p):
            line=line.strip()
            if not line: continue
            try: o=json.loads(line)
            except Exception: continue
            s=o.get("sku")
            if s: by_sku[s]=o
    except FileNotFoundError:
        pass
with open(final,"w") as f:
    for s in sorted(by_sku):
        f.write(json.dumps(by_sku[s])+"\n")
print(f"[merge] final unique SKUs = {len(by_sku)} -> {final}")
PY
log "MERGE done. final=$(wc -l < "$FINAL" 2>/dev/null) unique SKUs"
touch "$ST/ENRICH-DONE"
log "ALL DONE"