← back to Japan Enrich
sangetsu enricher: atomic temp+rename checkpoint + carry-forward resume (restart can't truncate baseline)
1aceaf0fb5f9e1e2b6f807620558265990e8dfd4 · 2026-07-01 13:34:32 -0700 · Steve Abrams
Files touched
Diff
commit 1aceaf0fb5f9e1e2b6f807620558265990e8dfd4
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Jul 1 13:34:32 2026 -0700
sangetsu enricher: atomic temp+rename checkpoint + carry-forward resume (restart can't truncate baseline)
---
enrich_sangetsu.py | 52 ++++++++++++++++++++++++++++++++++++++++++++--------
1 file changed, 44 insertions(+), 8 deletions(-)
diff --git a/enrich_sangetsu.py b/enrich_sangetsu.py
index 74cee39..5915c06 100644
--- a/enrich_sangetsu.py
+++ b/enrich_sangetsu.py
@@ -50,16 +50,50 @@ def main():
done += 1
if done % 500 == 0: print(f" fetched {done}/{len(jobs)} ({len(paths)} ok)", flush=True)
print(f"download complete: {len(paths)} images on disk", flush=True)
+
+ # Resume-safe: load any prior enrichment so a re-run carries it forward
+ # (skips the slow vision leg for patterns already done) and NEVER regresses.
+ existing = {}
+ if os.path.exists(OUT):
+ for l in open(OUT):
+ l = l.strip()
+ if not l: continue
+ try:
+ o = json.loads(l)
+ if o.get("mfr_sku"): existing[o["mfr_sku"]] = o
+ except Exception:
+ pass
+ print(f"carry-forward: {len(existing)} colorways already enriched", flush=True)
+
+ TMP = OUT + ".tmp"
+ def checkpoint(lines, processed):
+ # Atomic write of a SUPERSET: everything processed so far PLUS any prior
+ # entry not yet reprocessed. temp+rename => a mid-run death can't truncate.
+ leftover = [json.dumps(o, ensure_ascii=False) for s, o in existing.items() if s not in processed]
+ with open(TMP, "w") as f:
+ f.write("\n".join(lines + leftover))
+ if lines or leftover: f.write("\n")
+ os.replace(TMP, OUT)
+
# 2) enrich: vision once per pattern (rep image), Pillow per colorway
- w = open(OUT, "w"); n_pat = 0; n_sku = 0
+ out_lines = []; processed = set(); n_pat = 0; n_sku = 0; n_vision = 0
for r in rows:
skus = r.get("colorway_skus") or []
have = [(s, paths[norm_sku(s)]) for s in skus if norm_sku(s) in paths]
if not have: continue
- # representative vision call (first colorway with an image)
- style, material = vision_tags(have[0][1])
+ # reuse style/material from any already-enriched colorway (skip slow vision)
+ style = material = None
+ for s, _ in have:
+ e = (existing.get(s) or {}).get("enrich") or {}
+ if e.get("style") is not None or e.get("material") is not None:
+ style, material = e.get("style"), e.get("material"); break
+ if style is None and material is None:
+ style, material = vision_tags(have[0][1]); n_vision += 1
n_pat += 1
for s, p in have:
+ if s in existing:
+ out_lines.append(json.dumps(existing[s], ensure_ascii=False))
+ processed.add(s); n_sku += 1; continue
try:
hexes = hex_palette(p)
except Exception:
@@ -68,12 +102,14 @@ def main():
"style": style, "material": material,
"img": os.path.basename(p),
"engine": "local-hybrid:pillow+qwen2.5vl:7b(pattern-vision)"}
- w.write(json.dumps({"mfr_sku": s, "source": "sangetsu", "enrich": enrich}, ensure_ascii=False) + "\n")
- n_sku += 1
+ out_lines.append(json.dumps({"mfr_sku": s, "source": "sangetsu", "enrich": enrich}, ensure_ascii=False))
+ processed.add(s); n_sku += 1
if n_pat % 25 == 0:
- print(f" enriched {n_pat} patterns / {n_sku} colorway skus (last: {r.get('pattern')} -> {style}/{material})", flush=True)
- w.close()
- print(f"\nDONE: enriched {n_sku} colorway SKUs across {n_pat} patterns -> {OUT}", flush=True)
+ checkpoint(out_lines, processed)
+ print(f" enriched {n_pat} patterns / {n_sku} colorway skus ({n_vision} vision calls; last: {r.get('pattern')} -> {style}/{material})", flush=True)
+ checkpoint(out_lines, processed)
+ total = len(set(processed) | set(existing))
+ print(f"\nDONE: {n_sku} colorways this pass across {n_pat} patterns ({n_vision} vision calls); {total} total enriched -> {OUT}", flush=True)
if __name__ == "__main__":
main()
← 2d6bdd6 auto-save: 2026-07-01T12:12:34 (1 files) — out/officer-yolo-
·
back to Japan Enrich
·
enricher: drop unused imports (pyflakes clean) 0173358 →