← back to Dw Pairs Well
tools/pattern-id-titlefirst-dryrun.py
199 lines
#!/usr/bin/env python3
"""WS-1 TITLE-FIRST + IMAGE-CONFIRMED pattern_id derivation — DRY RUN (read-only, $0 local).
Per /dtd verdict (3/3 A, 2026-06-26): derive the grouping key TITLE-FIRST
(normalized title primary, mfr-root fallback for junk/empty titles), then
IMAGE-CONFIRM within each title group so two DIFFERENT designs that merely share
a title (e.g. "Allegro | Kravet" vs "Allegro | Lee Jofa") split apart, while real
colorways of one design (same pattern, different colors) stay merged.
NO WRITES. Reads the mirror + tools/pattern-id-by-dwsku.json (the CURRENT, mfr-first
authoritative keys) and prints a before/after diff so Steve can eyeball the change
before any gated re-backfill.
Input on stdin: TSV rows dw_sku <TAB> vendor <TAB> title <TAB> mfr_sku <TAB> dhash(16hex or '')
(dhash from image_hashes where status='success', joined on image_url)
Color-invariance note: the precomputed image_hashes.dhash is a single grayscale
dHash (already tone-insensitive for same-tone colorways). For light-on-dark vs
dark-on-light inversions, tools/pattern-image-verify.py downloads + adds the tonal
inverse; that finer check is reserved for flagged ambiguous groups, not this sweep.
"""
import sys, json, re, os
from collections import defaultdict, Counter
HAM_THRESH = int(os.environ.get("HAM_THRESH", "12")) # dHash Hamming <= => same pattern
MAP_PATH = os.path.join(os.path.dirname(__file__), "pattern-id-by-dwsku.json")
# ---- normalization (faithful port of server.js gNormTitle/gStripColors/gMfrRoot/gVendorSlug) ----
NOISE = ['wallcovering','wallcoverings','wallpaper','wall covering','fabric','fabrics',
'durable vinyl','vinyl','grasscloth','print','commercial','residential',
'type ii','type iii','mural','wall mural','peel and stick','peel-and-stick',
'self adhesive','self-adhesive','by phillipe romano','phillipe romano']
COLOR_WORDS = set(['light','dark','deep','pale','soft','warm','cool','muted','bright',
'antique','metallic','natural','neutral','multi','multicolor','multicolour',
'black','white','grey','gray','silver','gold','beige','cream','ivory','taupe',
'brown','tan','red','crimson','burgundy','pink','rose','blush','coral','orange',
'rust','terracotta','yellow','ochre','mustard','green','olive','sage','emerald',
'teal','aqua','turquoise','blue','navy','indigo','cobalt','azure','purple','violet',
'lilac','lavender','plum','mauve','charcoal','slate','sand','stone','smoke','linen',
'pearl','champagne','bronze','copper','platinum','marine','lawn','fog','snow',
'azalea','cloud','cobalt','concord','cyan','dandelion','mushroom','seaweed'])
def strip_vendor_suffix(title):
i = title.find('|')
return (title if i == -1 else title[:i]).strip()
def norm_title(title):
s = strip_vendor_suffix(title or '').lower()
for n in NOISE: s = s.replace(n, ' ')
s = re.sub(r'[^a-z0-9 ]+', ' ', s)
s = re.sub(r'\s+', ' ', s).strip()
return s
def strip_colors(normalized):
kept = [w for w in normalized.split(' ') if w and w not in COLOR_WORDS]
return ' '.join(kept).strip() or normalized
def mfr_root(raw):
if not raw: return None
s = str(raw).strip().upper()
if not s: return None
toks = [t for t in re.split(r'[-/_.\s]+', s) if t]
if len(toks) >= 2 and re.fullmatch(r'\d{1,3}[A-Z]?', toks[-1]):
return '-'.join(toks[:-1])
return '-'.join(toks)
def vendor_slug(v):
s = re.sub(r'[^a-z0-9]+', '-', (v or '').lower()).strip('-')
return s or 'no-vendor'
DENY_TITLES = set(['','various','assorted','sample','default','tbd','test','n a','na'])
def title_first_key(vendor, title, mfr):
"""TITLE-FIRST: normalized title primary; mfr-root only when title is junk/empty."""
vslug = vendor_slug(vendor)
t2 = strip_colors(norm_title(title))
if t2 and t2 not in DENY_TITLES and len(t2) >= 3:
return f"{vslug}::t2::{t2}"
root = mfr_root(mfr)
if root:
return f"{vslug}::mfr::{root}"
return f"{vslug}::t2::{t2 or 'ungrouped'}"
def ham_hex(a, b):
return bin(int(a, 16) ^ int(b, 16)).count("1")
# ---- load CURRENT (mfr-first) authoritative keys for the before/after diff ----
cur = {}
try:
raw = json.load(open(MAP_PATH))
for k, v in raw.items():
if v and v.get('pid'): cur[k] = v['pid']
except Exception as e:
print(f"WARN: could not load current map: {e}", file=sys.stderr)
# ---- read rows ----
rows = []
for line in sys.stdin:
p = line.rstrip('\n').split('\t')
if len(p) < 4: continue
dw, ven, title, mfr = p[0], p[1], p[2], p[3]
dh = p[4].strip() if len(p) >= 5 and p[4].strip() and len(p[4].strip()) == 16 else None
if not dw: continue
rows.append((dw, ven, title, mfr, dh))
# ---- stage 1: title-first key ----
for i, (dw, ven, title, mfr, dh) in enumerate(rows):
rows[i] = (dw, ven, title, mfr, dh, title_first_key(ven, title, mfr))
# ---- stage 2: image-confirm within each title key (split visual collisions) ----
# Greedy clustering on precomputed dHash within a title group. Members without a
# hash join the FIRST cluster (cannot be disconfirmed) and are counted as unconfirmed.
bykey = defaultdict(list)
for r in rows: bykey[r[5]].append(r)
final_key = {} # dw_sku -> final (image-confirmed) key
img_splits = 0 # title groups that split into >1 visual cluster
confirmed_skus = 0
unconfirmed_skus = 0
for key, members in bykey.items():
hashed = [m for m in members if m[4]]
nohash = [m for m in members if not m[4]]
if len(members) == 1 or len(hashed) <= 1:
for m in members: final_key[m[0]] = key
unconfirmed_skus += len(nohash)
confirmed_skus += len(hashed)
continue
# greedy clusters among hashed members
clusters = [] # list of (rep_hash, [members])
for m in hashed:
placed = False
for cl in clusters:
if ham_hex(m[4], cl[0]) <= HAM_THRESH:
cl[1].append(m); placed = True; break
if not placed:
clusters.append((m[4], [m]))
if len(clusters) == 1:
for m in members: final_key[m[0]] = key
confirmed_skus += len(hashed); unconfirmed_skus += len(nohash)
else:
img_splits += 1
# assign each visual cluster a distinct sub-key; unhashed members -> largest cluster
clusters.sort(key=lambda c: -len(c[1]))
for ci, (rep, cms) in enumerate(clusters):
sub = key if ci == 0 else f"{key}#v{ci+1}"
for m in cms: final_key[m[0]] = sub
for m in nohash: final_key[m[0]] = key
confirmed_skus += len(hashed); unconfirmed_skus += len(nohash)
# ---- metrics: before (current mfr-first) vs after (title-first + image) ----
def group_stats(keyfn):
k2title = defaultdict(set); vt2k = defaultdict(set)
for (dw, ven, title, mfr, dh, _tk) in rows:
k = keyfn(dw, ven, title, mfr)
if not k: continue
nt = strip_colors(norm_title(title))
if nt: k2title[k].add(nt); vt2k[(ven.lower(), nt)].add(k)
over = sum(1 for k, ts in k2title.items() if len(ts) > 1) # distinct designs lumped
under = sum(1 for vt, ks in vt2k.items() if len(ks) > 1) # one design fragmented
tiles = len(k2title)
return tiles, over, under
before = group_stats(lambda dw, ven, title, mfr: cur.get(dw))
after = group_stats(lambda dw, ven, title, mfr: final_key.get(dw))
print("="*70)
print(f"TITLE-FIRST + IMAGE-CONFIRMED pattern_id — DRY RUN (no writes)")
print(f"active rows: {len(rows)} · HAM_THRESH={HAM_THRESH}")
print(f"image coverage: {confirmed_skus} hashed / {unconfirmed_skus} title-only "
f"({confirmed_skus*100//max(1,len(rows))}% image-confirmable)")
print(f"image-driven splits (title groups broken into visual clusters): {img_splits}")
print("="*70)
print(f"{'metric':<42}{'BEFORE (mfr-first)':>16}{'AFTER':>12}")
print(f"{' distinct pattern tiles':<42}{before[0]:>16}{after[0]:>12}")
print(f"{' OVER-merge keys (diff designs lumped)':<42}{before[1]:>16}{after[1]:>12}")
print(f"{' UNDER-merge groups (1 design split)':<42}{before[2]:>16}{after[2]:>12}")
print("="*70)
# ---- worst-offender spotlight ----
def show(label, match):
cur_keys = Counter(); new_keys = Counter()
for (dw, ven, title, mfr, dh, _tk) in rows:
if not match(ven, title): continue
cur_keys[cur.get(dw, '(none)')] += 1
new_keys[final_key.get(dw, '(none)')] += 1
print(f"\n— {label} —")
print(f" BEFORE: {len(cur_keys)} key(s) -> " +
", ".join(f"{k.split('::')[-1]}×{n}" for k, n in cur_keys.most_common(4)))
print(f" AFTER : {len(new_keys)} key(s) -> " +
", ".join(f"{k.split('::')[-1]}×{n}" for k, n in new_keys.most_common(4)))
show("Yakatore (was fragmented 7 ways)", lambda v, t: 'phillipe' in v.lower() and 'yakatore' in t.lower())
show("Coordonne Durable (was 366 keys)", lambda v, t: 'coordonn' in t.lower())
show("Ultrasuede 5538 (was 93 titles merged)", lambda v, t: 'ultrasuede' in v.lower())
show("Phillipe Romano CORK (was 16 designs merged)",
lambda v, t: 'phillipe' in v.lower() and 'cork' in t.lower() and 'corka' not in t.lower())
show("La Corka (real 11-colorway pattern — must stay merged)",
lambda v, t: 'phillipe' in v.lower() and 'corka' in t.lower())