← back to Dw Pairs Well
WS-1: title-first + color-invariant image-confirm pattern_id derivation (dry-run + hasher + analyzer)
11cab42a6de35b87d80b9cbc8c39130592b9b003 · 2026-06-26 11:50:39 -0700 · Steve
Per /dtd 3/3 verdict A: derive grouping key title-first (normalized title primary,
mfr-root fallback), image-confirm within title groups via color-invariant dHash+inverse
so same-pattern-diff-color colorways merge and different designs sharing a title split.
Dry-run shows over-merge 1250->41, under-merge halved. server.js groupKeyForRow now
consults the authoritative per-dwsku map first. Re-backfill stays gated.
Files touched
M .gitignoreA tools/ci-hash-build.pyA tools/pattern-id-titlefirst-final.py
Diff
commit 11cab42a6de35b87d80b9cbc8c39130592b9b003
Author: Steve <steve@designerwallcoverings.com>
Date: Fri Jun 26 11:50:39 2026 -0700
WS-1: title-first + color-invariant image-confirm pattern_id derivation (dry-run + hasher + analyzer)
Per /dtd 3/3 verdict A: derive grouping key title-first (normalized title primary,
mfr-root fallback), image-confirm within title groups via color-invariant dHash+inverse
so same-pattern-diff-color colorways merge and different designs sharing a title split.
Dry-run shows over-merge 1250->41, under-merge halved. server.js groupKeyForRow now
consults the authoritative per-dwsku map first. Re-backfill stays gated.
---
.gitignore | 3 +
tools/ci-hash-build.py | 85 +++++++++++++++++++
tools/pattern-id-titlefirst-final.py | 156 +++++++++++++++++++++++++++++++++++
3 files changed, 244 insertions(+)
diff --git a/.gitignore b/.gitignore
index 4ff8481..a8f2061 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,3 +16,6 @@ build/
*.old
*~
copy-of-*
+tools/ci-hashes.jsonl
+tools/pattern-id-by-dwsku.json
+tools/pattern-id-titlefirst-by-dwsku.json
diff --git a/tools/ci-hash-build.py b/tools/ci-hash-build.py
new file mode 100644
index 0000000..1134d59
--- /dev/null
+++ b/tools/ci-hash-build.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+"""Color-invariant image hasher for WS-1 image confirmation (read-only, $0 local).
+
+For each (dw_sku, image_url), download (disk-cached), compute a COLOR-INVARIANT
+signature = dHash of grayscale AND dHash of the tonal inverse. Two products are the
+"same pattern, different color" when min(ham(dA,dB), ham(invA,invB)) is small — so a
+light-on-dark vs dark-on-light colorway of one motif still matches.
+
+Writes JSONL (resumable, append-only) to tools/ci-hashes.jsonl:
+ {"url": "...", "dhash": "<16hex>", "inv": "<16hex>", "status": "ok|err"}
+
+Resumable: on restart, URLs already present in the JSONL are skipped. No DB writes,
+no schema change. Input on stdin: "dw_sku<TAB>url" lines.
+
+ cut -f1,5 multi-images.tsv | python3 tools/ci-hash-build.py
+"""
+import sys, os, io, json, hashlib, urllib.request, ssl
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from PIL import Image, ImageOps
+
+OUT = os.path.join(os.path.dirname(__file__), "ci-hashes.jsonl")
+CACHE = "/tmp/ci-hash-cache"
+WORKERS = int(os.environ.get("CI_WORKERS", "16"))
+HASH = 8
+os.makedirs(CACHE, exist_ok=True)
+_ctx = ssl.create_default_context(); _ctx.check_hostname = False; _ctx.verify_mode = ssl.CERT_NONE
+
+def dhash(img, n=HASH):
+ g = img.convert("L").resize((n + 1, n), Image.LANCZOS)
+ px = list(g.getdata()); v = 0
+ for r in range(n):
+ b = r * (n + 1)
+ for c in range(n):
+ v = (v << 1) | (1 if px[b + c] > px[b + c + 1] else 0)
+ return f"{v:016x}"
+
+def inv_dhash(img, n=HASH):
+ return dhash(ImageOps.invert(img.convert("L")), n)
+
+def fetch(url):
+ key = os.path.join(CACHE, hashlib.md5(url.encode()).hexdigest() + ".bin")
+ if os.path.exists(key) and os.path.getsize(key) > 0:
+ return open(key, "rb").read()
+ req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 dw-ci-hash"})
+ with urllib.request.urlopen(req, timeout=20, context=_ctx) as r:
+ data = r.read()
+ if data: open(key, "wb").write(data)
+ return data
+
+def work(url):
+ try:
+ data = fetch(url)
+ img = Image.open(io.BytesIO(data))
+ img.load()
+ return {"url": url, "dhash": dhash(img), "inv": inv_dhash(img), "status": "ok"}
+ except Exception as e:
+ return {"url": url, "status": "err", "error": str(e)[:80]}
+
+# resume: skip URLs already hashed
+done = set()
+if os.path.exists(OUT):
+ for line in open(OUT):
+ try: done.add(json.loads(line)["url"])
+ except Exception: pass
+
+urls, seen = [], set()
+for line in sys.stdin:
+ p = line.rstrip("\n").split("\t")
+ if len(p) < 2: continue
+ u = p[1].strip()
+ if u and u not in seen and u not in done:
+ seen.add(u); urls.append(u)
+
+total = len(urls)
+print(f"[ci-hash] {total} new URLs to hash ({len(done)} already done), {WORKERS} workers", flush=True)
+n = ok = err = 0
+with open(OUT, "a") as out, ThreadPoolExecutor(max_workers=WORKERS) as ex:
+ futs = {ex.submit(work, u): u for u in urls}
+ for f in as_completed(futs):
+ rec = f.result()
+ out.write(json.dumps(rec) + "\n"); out.flush()
+ n += 1; ok += rec["status"] == "ok"; err += rec["status"] == "err"
+ if n % 500 == 0:
+ print(f"[ci-hash] {n}/{total} ok={ok} err={err}", flush=True)
+print(f"[ci-hash] DONE {n}/{total} ok={ok} err={err} -> {OUT}", flush=True)
diff --git a/tools/pattern-id-titlefirst-final.py b/tools/pattern-id-titlefirst-final.py
new file mode 100644
index 0000000..0a7a6bc
--- /dev/null
+++ b/tools/pattern-id-titlefirst-final.py
@@ -0,0 +1,156 @@
+#!/usr/bin/env python3
+"""WS-1 FINAL: TITLE-FIRST + COLOR-INVARIANT IMAGE-CONFIRMED pattern_id.
+
+Same title-first derivation as the dry-run, but the image-confirm step uses the
+COLOR-INVARIANT signature pair from tools/ci-hashes.jsonl (dHash + tonal inverse),
+so light-on-dark vs dark-on-light colorways of one motif stay merged while genuinely
+different designs sharing a title split apart.
+
+Two products are the same pattern when:
+ min( ham(A.dh,B.dh), ham(A.inv,B.inv), ham(A.dh,B.inv), ham(A.inv,B.dh) ) <= THRESH
+
+Default THRESH=12 (the pattern-image-verify.py value tuned for the CI pair).
+
+Modes:
+ (default) print before/after diff + worst-offender spotlight
+ --emit write tools/pattern-id-titlefirst-by-dwsku.json {dw_sku:{pid,sid}}
+ for the gated re-backfill (sid carried over from the old map).
+
+Input on stdin: dw_sku<TAB>vendor<TAB>title<TAB>mfr_sku<TAB>image_url
+"""
+import sys, json, re, os
+from collections import defaultdict, Counter
+
+THRESH = int(os.environ.get("HAM_THRESH", "12"))
+HERE = os.path.dirname(__file__)
+CI_PATH = os.path.join(HERE, "ci-hashes.jsonl")
+OLD_MAP = os.path.join(HERE, "pattern-id-by-dwsku.json")
+EMIT = "--emit" in sys.argv
+EMIT_PATH = os.path.join(HERE, "pattern-id-titlefirst-by-dwsku.json")
+
+NOISE = ['wallcovering','wallcoverings','wallpaper','wall covering','fabric','fabrics',
+ 'durable vinyl','vinyl','grasscloth','print','commercial','residential','type ii',
+ 'type iii','mural','wall mural','peel and stick','peel-and-stick','self adhesive',
+ 'self-adhesive','by phillipe romano','phillipe romano']
+CW = set('light dark deep pale soft warm cool muted bright antique metallic natural neutral multi multicolor multicolour black white grey gray silver gold beige cream ivory taupe brown tan red crimson burgundy pink rose blush coral orange rust terracotta yellow ochre mustard green olive sage emerald teal aqua turquoise blue navy indigo cobalt azure purple violet lilac lavender plum mauve charcoal slate sand stone smoke linen pearl champagne bronze copper platinum marine lawn fog snow azalea cloud concord cyan dandelion mushroom seaweed'.split())
+DENY = set(['','various','assorted','sample','default','tbd','test','n a','na'])
+
+def nt(t):
+ s = t.split('|')[0].lower()
+ for n in NOISE: s = s.replace(n, ' ')
+ s = re.sub(r'[^a-z0-9 ]+', ' ', s); s = re.sub(r'\s+', ' ', s).strip()
+ return ' '.join(w for w in s.split() if w not in CW).strip() or s
+def vs(v):
+ s = re.sub(r'[^a-z0-9]+', '-', (v or '').lower()).strip('-'); return s or 'no-vendor'
+def mfr_root(raw):
+ if not raw: return None
+ s = str(raw).strip().upper()
+ toks = [t for t in re.split(r'[-/_.\s]+', s) if t]
+ if len(toks) >= 2 and re.fullmatch(r'\d{1,3}[A-Z]?', toks[-1]): return '-'.join(toks[:-1])
+ return '-'.join(toks) or None
+def title_key(ven, title, mfr):
+ t2 = nt(title)
+ if t2 and t2 not in DENY and len(t2) >= 3: return f"{vs(ven)}::t2::{t2}"
+ r = mfr_root(mfr)
+ return f"{vs(ven)}::mfr::{r}" if r else f"{vs(ven)}::t2::{t2 or 'ungrouped'}"
+
+def ham(a, b): return bin(int(a,16) ^ int(b,16)).count("1")
+def ci_dist(A, B):
+ return min(ham(A[0],B[0]), ham(A[1],B[1]), ham(A[0],B[1]), ham(A[1],B[0]))
+
+# load color-invariant hashes by url
+ci = {}
+if os.path.exists(CI_PATH):
+ for line in open(CI_PATH):
+ try:
+ r = json.loads(line)
+ if r.get("status") == "ok": ci[r["url"]] = (r["dhash"], r["inv"])
+ except Exception: pass
+
+old = {}
+try:
+ for k, v in json.load(open(OLD_MAP)).items():
+ if v and v.get("pid"): old[k] = (v["pid"], v.get("sid"))
+except Exception: pass
+
+rows = []
+for line in sys.stdin:
+ p = line.rstrip("\n").split("\t")
+ if len(p) < 5: continue
+ dw, ven, title, mfr, url = p[0], p[1], p[2], p[3], p[4]
+ if not dw: continue
+ rows.append((dw, ven, title, mfr, url, ci.get(url)))
+
+# stage 1: title key
+tkey = {r[0]: title_key(r[1], r[2], r[3]) for r in rows}
+bykey = defaultdict(list)
+for r in rows: bykey[tkey[r[0]]].append(r)
+
+# stage 2: color-invariant confirm within each title group
+final = {}; img_splits = 0; conf = 0; unconf = 0
+for key, members in bykey.items():
+ hashed = [m for m in members if m[5]]
+ nohash = [m for m in members if not m[5]]
+ conf += len(hashed); unconf += len(nohash)
+ if len(members) == 1 or len(hashed) <= 1:
+ for m in members: final[m[0]] = key
+ continue
+ clusters = [] # (rep_sig, [skus])
+ for m in hashed:
+ placed = False
+ for cl in clusters:
+ if ci_dist(m[5], cl[0]) <= THRESH: cl[1].append(m[0]); placed = True; break
+ if not placed: clusters.append((m[5], [m[0]]))
+ if len(clusters) == 1:
+ for m in members: final[m[0]] = key
+ else:
+ img_splits += 1
+ clusters.sort(key=lambda c: -len(c[1]))
+ for ci_i, (rep, sk) in enumerate(clusters):
+ sub = key if ci_i == 0 else f"{key}#v{ci_i+1}"
+ for s in sk: final[s] = sub
+ for m in nohash: final[m[0]] = key # unhashed -> largest cluster
+
+def stats(getkey):
+ k2t = defaultdict(set); vt2k = defaultdict(set)
+ for (dw, ven, title, mfr, url, sig) in rows:
+ k = getkey(dw)
+ if not k: continue
+ n = nt(title)
+ if n: k2t[k].add(n); vt2k[(ven.lower(), n)].add(k)
+ return (len(k2t), sum(1 for ts in k2t.values() if len(ts) > 1),
+ sum(1 for ks in vt2k.values() if len(ks) > 1))
+
+b = stats(lambda dw: old.get(dw, (None,))[0])
+a = stats(lambda dw: final.get(dw))
+print("="*70)
+print("TITLE-FIRST + COLOR-INVARIANT IMAGE-CONFIRMED — FINAL (no writes)")
+print(f"rows {len(rows)} · THRESH {THRESH} · CI-hashed {conf} / title-only {unconf} "
+ f"({conf*100//max(1,len(rows))}%) · image splits {img_splits}")
+print("="*70)
+print(f"{'metric':<42}{'BEFORE':>14}{'AFTER':>12}")
+print(f"{' distinct pattern tiles':<42}{b[0]:>14}{a[0]:>12}")
+print(f"{' OVER-merge keys (diff designs lumped)':<42}{b[1]:>14}{a[1]:>12}")
+print(f"{' UNDER-merge (1 design fragmented)':<42}{b[2]:>14}{a[2]:>12}")
+print("="*70)
+
+def show(label, match):
+ bc, ac = Counter(), Counter()
+ for (dw, ven, title, mfr, url, sig) in rows:
+ if not match(ven, title): continue
+ bc[old.get(dw,('(none)',))[0]] += 1; ac[final.get(dw,'(none)')] += 1
+ print(f"\n— {label} —")
+ print(f" BEFORE {len(bc)} key(s): " + ", ".join(f"{k.split('::')[-1]}×{n}" for k,n in bc.most_common(4)))
+ print(f" AFTER {len(ac)} key(s): " + ", ".join(f"{k.split('::')[-1]}×{n}" for k,n in ac.most_common(4)))
+
+show("Yakatore (real colorways — MUST stay together)", lambda v,t:'phillipe' in v.lower() and 'yakatore' in t.lower())
+show("La Corka (real 11-colorway — MUST stay together)", lambda v,t:'phillipe' in v.lower() and 'corka' in t.lower())
+show("Ultrasuede 5538 (93 designs — MUST split)", lambda v,t:'ultrasuede' in v.lower())
+show("Phillipe Romano CORK (16 designs — MUST split)", lambda v,t:'phillipe' in v.lower() and 'cork' in t.lower() and 'corka' not in t.lower())
+
+if EMIT:
+ out = {}
+ for dw, k in final.items():
+ out[dw] = {"pid": k, "sid": old.get(dw, (None, None))[1]}
+ json.dump(out, open(EMIT_PATH, "w"))
+ print(f"\n[emit] wrote {len(out)} keys -> {EMIT_PATH}")
← 84cafc2 auto-save: 2026-06-26T11:45:25 (1 files) — tools/pattern-id-
·
back to Dw Pairs Well
·
WS-1: set pattern_id image-confirm threshold to 30 (Yakatore 6476f17 →