← back to Japan Enrich
lilycolor title_en synthesizer + per-item cleanup pass (draft EN titles, $0 local qwen)
2a647d232f74292ebc2f425f8a94a0f4e5de82ae · 2026-07-01 14:59:39 -0700 · Steve Abrams
Files touched
A enrich_lilycolor_titles.pyA fix_lilycolor_titles.py
Diff
commit 2a647d232f74292ebc2f425f8a94a0f4e5de82ae
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Wed Jul 1 14:59:39 2026 -0700
lilycolor title_en synthesizer + per-item cleanup pass (draft EN titles, $0 local qwen)
---
enrich_lilycolor_titles.py | 91 ++++++++++++++++++++++++++++++++++++++++++++++
fix_lilycolor_titles.py | 54 +++++++++++++++++++++++++++
2 files changed, 145 insertions(+)
diff --git a/enrich_lilycolor_titles.py b/enrich_lilycolor_titles.py
new file mode 100644
index 0000000..89dcaa8
--- /dev/null
+++ b/enrich_lilycolor_titles.py
@@ -0,0 +1,91 @@
+#!/usr/bin/env python3
+"""
+Lilycolor title_en synthesizer — DRAFT English titles ($0 local, qwen3:14b).
+
+title_ja is structured "PatternName｜SKU-range｜suffix" and the pattern name is
+mostly katakana loanwords (ブークレ=Bouclé, ベロア=Velour). We:
+ 1) extract the pattern name = title_ja.split('｜')[0]
+ 2) batch-translate the UNIQUE set of names via qwen3:14b (local, free)
+ 3) rows with no title_ja get a derived "Lilycolor <Collection>" label
+Every synthesized title is marked title_en_source so it's clearly a DRAFT —
+these are activation-gated and NOT customer-live until Steve reviews.
+
+Atomic temp+rename write. Idempotent: skips rows that already have title_en.
+"""
+import os, json, urllib.request
+
+SRC = "staging/lilycolor-unified-staging.jsonl"
+OLLAMA = "http://127.0.0.1:11434/api/generate"
+MODEL = "qwen2.5vl:7b" # already-loaded; avoids OLLAMA_MAX_LOADED swap + qwen3 <think> stall
+PIPE = "｜" # U+FF5C fullwidth pipe
+
+CATALOG_LABEL = {
+ "LIGHT": "Light", "WILL": "Will", "V-wall": "V-Wall",
+ "MATERIALS": "Materials", "ImportSelection": "Import Selection",
+}
+
+def qwen(prompt):
+ body = json.dumps({"model": MODEL, "stream": False, "prompt": prompt,
+ "options": {"temperature": 0}}).encode()
+ req = urllib.request.Request(OLLAMA, data=body, headers={"Content-Type": "application/json"})
+ return json.loads(urllib.request.urlopen(req, timeout=180).read())["response"]
+
+def translate_batch(names):
+ numbered = "\n".join(f"{i+1}. {n}" for i, n in enumerate(names))
+ prompt = ("Translate these Japanese wallcovering pattern names to concise English "
+ "design terms (1-3 words each; keep proper design vocabulary like Boucle, "
+ "Velour, Damask). Output ONLY a JSON array of strings in the same order, "
+ "no commentary, no <think>.\n" + numbered)
+ raw = qwen(prompt)
+ # strip any <think>…</think> and grab the JSON array
+ if "</think>" in raw: raw = raw.split("</think>")[-1]
+ s, e = raw.find("["), raw.rfind("]")
+ arr = json.loads(raw[s:e+1])
+ if len(arr) != len(names):
+ raise ValueError(f"count mismatch {len(arr)} != {len(names)}")
+ return [str(x).strip() for x in arr]
+
+def main():
+ rows = [json.loads(l) for l in open(SRC) if l.strip()]
+ # unique katakana pattern names to translate
+ def pname(r):
+ t = r.get("title_ja")
+ return t.split(PIPE)[0].strip() if t else None
+ uniq = sorted({pname(r) for r in rows if pname(r)})
+ print(f"{len(rows)} rows | {len(uniq)} unique JP pattern names to translate", flush=True)
+
+ mapping = {}
+ B = 25
+ for i in range(0, len(uniq), B):
+ chunk = uniq[i:i+B]
+ try:
+ en = translate_batch(chunk)
+ except Exception as ex:
+ print(f" batch {i//B} retry ({ex})", flush=True)
+ try: en = translate_batch(chunk)
+ except Exception as ex2:
+ print(f" batch {i//B} FAILED, romaji-fallback ({ex2})", flush=True)
+ en = chunk # last resort: keep original (better than blank)
+ for jp, e in zip(chunk, en): mapping[jp] = e
+ print(f" translated {min(i+B,len(uniq))}/{len(uniq)}", flush=True)
+
+ n_mt = n_derived = n_skip = 0
+ for r in rows:
+ if r.get("title_en"): # idempotent
+ n_skip += 1; continue
+ p = pname(r)
+ if p and mapping.get(p):
+ r["title_en"] = mapping[p]; r["title_en_source"] = "qwen3-mt-draft"; n_mt += 1
+ else:
+ lbl = CATALOG_LABEL.get(r.get("source_catalog"), r.get("source_catalog") or "Lilycolor")
+ r["title_en"] = f"Lilycolor {lbl} — {r.get('mfr_sku')}"
+ r["title_en_source"] = "derived"; n_derived += 1
+
+ TMP = SRC + ".tmp"
+ with open(TMP, "w") as f:
+ for r in rows: f.write(json.dumps(r, ensure_ascii=False) + "\n")
+ os.replace(TMP, SRC)
+ print(f"\nDONE: title_en set on {n_mt} (mt-draft) + {n_derived} (derived); {n_skip} already had one -> {SRC}", flush=True)
+
+if __name__ == "__main__":
+ main()
diff --git a/fix_lilycolor_titles.py b/fix_lilycolor_titles.py
new file mode 100644
index 0000000..436dccc
--- /dev/null
+++ b/fix_lilycolor_titles.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""
+Cleanup pass for enrich_lilycolor_titles.py: any title_en that still contains
+Japanese (a failed batch fell back to the raw katakana name) gets re-translated
+ONE name at a time — per-item calls can't count-mismatch, so they're reliable.
+Atomic temp+rename write. $0 local (qwen2.5vl:7b, already loaded).
+"""
+import os, json, re, urllib.request
+
+SRC = "staging/lilycolor-unified-staging.jsonl"
+OLLAMA = "http://127.0.0.1:11434/api/generate"
+MODEL = "qwen2.5vl:7b"
+CJK = re.compile(r'[-ゟ゠-ヿ一-鿿]')
+
+def translate_one(name):
+ prompt = ("Translate this Japanese wallcovering pattern name to a concise English "
+ "design term (1-3 words, e.g. Boucle, Velour, Damask, Stucco). "
+ "Output ONLY the English term, nothing else.\n" + name)
+ body = json.dumps({"model": MODEL, "stream": False, "prompt": prompt,
+ "options": {"temperature": 0, "num_predict": 24}}).encode()
+ req = urllib.request.Request(OLLAMA, data=body, headers={"Content-Type": "application/json"})
+ raw = json.loads(urllib.request.urlopen(req, timeout=60).read())["response"]
+ if "</think>" in raw: raw = raw.split("</think>")[-1]
+ # first non-empty line, strip quotes/punctuation
+ line = next((l.strip().strip('"\'`.').strip() for l in raw.splitlines() if l.strip()), "")
+ return line
+
+def main():
+ rows = [json.loads(l) for l in open(SRC) if l.strip()]
+ need = [r for r in rows if r.get("title_en") and CJK.search(r["title_en"])]
+ uniq = sorted({r["title_en"] for r in need})
+ print(f"{len(need)} rows still Japanese across {len(uniq)} unique names", flush=True)
+ fixed = {}
+ for i, jp in enumerate(uniq):
+ try:
+ en = translate_one(jp)
+ if en and not CJK.search(en): fixed[jp] = en
+ except Exception as ex:
+ print(f" '{jp}' failed ({ex})", flush=True)
+ if (i + 1) % 25 == 0: print(f" {i+1}/{len(uniq)}", flush=True)
+ n = 0
+ for r in rows:
+ t = r.get("title_en")
+ if t and t in fixed:
+ r["title_en"] = fixed[t]; r["title_en_source"] = "qwen-mt-draft-peritem"; n += 1
+ TMP = SRC + ".tmp"
+ with open(TMP, "w") as f:
+ for r in rows: f.write(json.dumps(r, ensure_ascii=False) + "\n")
+ os.replace(TMP, SRC)
+ remain = sum(1 for r in rows if r.get("title_en") and CJK.search(r["title_en"]))
+ print(f"\nDONE: fixed {n} rows; {remain} still Japanese (untranslatable/failed) -> {SRC}", flush=True)
+
+if __name__ == "__main__":
+ main()
← 5a103b4 chore: lint (clean) + refactor (MIN_IMAGE_BYTES const), v0.4
·
back to Japan Enrich
·
auto-save: 2026-07-01T15:37:13 (1 files) — staging/lilycolor e83f5b5 →