← back to Designer Wallcoverings
merge-staging: idempotent join of unified specs + local enrichment + EN titles into publish-ready records (gates untouched)
b489cfc0f9b8697f497ca84644f6bf4db89e3c20 · 2026-07-01 11:34:54 -0700 · Steve
Files touched
A onboarding/sangetsu-lilycolor/scripts/merge-staging.py
Diff
commit b489cfc0f9b8697f497ca84644f6bf4db89e3c20
Author: Steve <steve@designerwallcoverings.com>
Date: Wed Jul 1 11:34:54 2026 -0700
merge-staging: idempotent join of unified specs + local enrichment + EN titles into publish-ready records (gates untouched)
---
.../sangetsu-lilycolor/scripts/merge-staging.py | 135 +++++++++++++++++++++
1 file changed, 135 insertions(+)
diff --git a/onboarding/sangetsu-lilycolor/scripts/merge-staging.py b/onboarding/sangetsu-lilycolor/scripts/merge-staging.py
new file mode 100644
index 00000000..c9b303ef
--- /dev/null
+++ b/onboarding/sangetsu-lilycolor/scripts/merge-staging.py
@@ -0,0 +1,135 @@
+#!/usr/bin/env python3
+"""
+merge-staging.py — assemble the publish-ready Lilycolor record set.
+
+OFFLINE / non-gated prep. Left-joins onto the 2,543 unified-staging records
+(the publishable universe, keyed by mfr_sku):
+ - design enrichment (enrichment-full-063026A.jsonl, key: sku)
+ - EN titles (lilycolor-titles-en-063026A.jsonl, key: sku)
+
+Preserves every gating flag exactly as-is (cost_confirmed, activation_ready,
+status) — this script NEVER flips a gate. It only co-locates the staged data
+so that when Steve clears the gates (DWGO prefix, cost-sourcing, patch) the
+publish step reads one file.
+
+Idempotent + resumable: pure re-read of source files, deterministic rebuild.
+Re-run any time to fold in more enrichment/translation as the jobs progress.
+"""
+import json
+import os
+import sys
+
+HERE = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+STAGING = os.path.join(HERE, "staging")
+
+UNIFIED = os.path.join(STAGING, "lilycolor-unified-staging.jsonl")
+ENRICH = os.path.join(STAGING, "enrichment-full-063026A.jsonl")
+TITLES = os.path.join(STAGING, "lilycolor-titles-en-063026A.jsonl")
+
+OUT = os.path.join(STAGING, "lilycolor-merged-063026A.jsonl")
+ORPHANS = os.path.join(STAGING, "lilycolor-merge-orphans-063026A.json")
+
+# enrichment fields to graft under design.* (ground-truth + local-vision)
+ENRICH_FIELDS = [
+ "hex", "bg_hex", "bg_name", "dominant_hex", "colorway_name",
+ "vl_color_names", "styles", "material", "patterns", "imageType",
+ "description", "_provider", "_cost",
+]
+# EN-title fields to graft (JA stays untouched on the base record)
+TITLE_FIELDS = ["title_en", "pattern_ja", "pattern_en", "suffix_en"]
+
+
+def load_by_key(path, key):
+ """Return {keyval: record}; last write wins on dup key."""
+ out = {}
+ if not os.path.exists(path):
+ return out
+ with open(path) as f:
+ for line in f:
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ d = json.loads(line)
+ except json.JSONDecodeError:
+ continue
+ k = d.get(key)
+ if k:
+ out[k] = d
+ return out
+
+
+def main():
+ enrich = load_by_key(ENRICH, "sku")
+ titles = load_by_key(TITLES, "sku")
+
+ n_out = 0
+ n_enriched = 0
+ n_titled = 0
+ n_both = 0
+ used_enrich = set()
+ used_titles = set()
+
+ with open(UNIFIED) as fin, open(OUT, "w") as fout:
+ for line in fin:
+ line = line.strip()
+ if not line:
+ continue
+ rec = json.loads(line)
+ sku = rec.get("mfr_sku")
+
+ e = enrich.get(sku)
+ t = titles.get(sku)
+
+ if e:
+ used_enrich.add(sku)
+ design = {k: e.get(k) for k in ENRICH_FIELDS if e.get(k) is not None}
+ rec["design"] = design # ground-truth hex + local-vision semantics
+ rec["_enriched"] = True
+ n_enriched += 1
+ else:
+ rec["_enriched"] = False
+
+ if t:
+ used_titles.add(sku)
+ # graft EN title fields WITHOUT clobbering the JA base title
+ for k in TITLE_FIELDS:
+ v = t.get(k)
+ if v is not None:
+ # translated EN title lands as title_en (base had it empty/JA-derived)
+ rec[k] = v
+ rec["_titled_en"] = True
+ n_titled += 1
+ else:
+ rec["_titled_en"] = False
+
+ if e and t:
+ n_both += 1
+
+ # gating flags are passed through verbatim — assert we never touched them
+ fout.write(json.dumps(rec, ensure_ascii=False) + "\n")
+ n_out += 1
+
+ # orphans: enrichment/title rows whose sku has no unified spec record
+ enrich_orphans = sorted(set(enrich) - used_enrich)
+ title_orphans = sorted(set(titles) - used_titles)
+ with open(ORPHANS, "w") as f:
+ json.dump({
+ "enrich_orphans": enrich_orphans, # image-only SKUs, no spec → cannot publish
+ "title_orphans": title_orphans,
+ "note": "orphans have translated/enriched data but no unified spec row; "
+ "expected since Henry holds 2,634 swatches vs 2,543 spec'd SKUs.",
+ }, f, indent=2, ensure_ascii=False)
+
+ print(f"merged records written : {n_out} -> {os.path.relpath(OUT, HERE)}")
+ print(f" with design enrichment: {n_enriched} ({100*n_enriched//n_out}%)")
+ print(f" with EN title : {n_titled} ({100*n_titled//n_out}%)")
+ print(f" with BOTH : {n_both}")
+ print(f"orphans (enrich/title) : {len(enrich_orphans)}/{len(title_orphans)}"
+ f" -> {os.path.relpath(ORPHANS, HERE)}")
+ # sanity: gates must all still be closed at this staging phase
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
← 2cc69d31 chore: enrichment finisher + retry-errors self-heal scripts,
·
back to Designer Wallcoverings
·
auto-save: 2026-07-01T11:42:20 (5 files) — pending-approval/ e1dcbbd9 →