← back to Designer Wallcoverings
onboarding/sangetsu-lilycolor/scripts/translate-titles.py
264 lines
#!/usr/bin/env python3
"""
translate-titles.py — Stage English translations of Lilycolor Japanese product
titles ALONGSIDE the Japanese source. READ-ONLY against the JA source: never
mutates staging/lilycolor-unified-staging.jsonl.
- Reads each record from the unified staging file, extracts `title_ja`.
- The JA title is shaped: <pattern-name>｜<sku-range>｜<suffix>
e.g. 山ぶどう｜LL6020・6021｜サンプル
Only the pattern-name segment (before the first ｜) carries meaning worth
translating; the SKU range is preserved as-is and the boilerplate suffix
(サンプル=Sample, のり付き壁紙=Pre-Pasted Wallcovering) is normalized.
- Calls local ollama qwen2.5vl:7b (localhost:11434) to translate the pattern
name into natural English Title Case. $0 (local). No paid APIs.
- Writes staging/lilycolor-titles-en-063026A.jsonl with:
{sku, title_ja, title_en, pattern_ja, pattern_en, suffix_en}
where title_en is the reassembled English title.
- RESUMABLE: skips any SKU already present in the output file.
- Model errors are logged + skipped (SKU not written; picked up on next run).
Standing rules enforced:
* The word "Wallpaper" is BANNED -> "Wallcovering".
* Title Case output.
* Do NOT invent pattern names: bare SKU/code stays faithful.
"""
import json
import os
import re
import time
import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.dirname(HERE)
SRC = os.path.join(ROOT, "staging", "lilycolor-unified-staging.jsonl")
OUT = os.path.join(ROOT, "staging", "lilycolor-titles-en-063026A.jsonl")
LOG = os.path.join(ROOT, "staging", "translate-titles.log")
OLLAMA = "http://localhost:11434/api/generate"
MODEL = "qwen2.5vl:7b"
# Boilerplate suffix map (deterministic; no model needed for these).
SUFFIX_MAP = {
"サンプル": "Sample",
"のり付き壁紙": "Pre-Pasted Wallcovering",
"のり無し壁紙": "Unpasted Wallcovering",
}
BANNED = re.compile(r"\bwallpaper(s)?\b", re.IGNORECASE)
# Small skip-lists: pure-ASCII / bare-code pattern names need no translation.
ASCII_ONLY = re.compile(r"^[\x00-\x7F]+$")
def log(msg):
line = f"{time.strftime('%Y-%m-%dT%H:%M:%S')} {msg}"
print(line, flush=True)
with open(LOG, "a", encoding="utf-8") as f:
f.write(line + "\n")
def title_case(s):
"""Title Case that leaves already-capitalized acronyms/codes alone and
keeps small articles/preps lowercase unless first word."""
small = {"a", "an", "the", "and", "but", "or", "of", "in", "on", "at",
"for", "to", "with", "by"}
words = s.split()
out = []
for i, w in enumerate(words):
# preserve tokens that already contain a digit or are ALLCAPS codes
if re.search(r"\d", w) or (w.isupper() and len(w) > 1):
out.append(w)
elif i > 0 and w.lower() in small:
out.append(w.lower())
else:
out.append(w[:1].upper() + w[1:])
return " ".join(out)
def clean_en(s):
s = s.strip().strip('"').strip("'").strip()
# collapse whitespace/newlines
s = re.sub(r"\s+", " ", s)
# strip a leading "English:"/"->" echo if the model repeats it
s = re.sub(r"^(english|translation)\s*[:\-]\s*", "", s, flags=re.IGNORECASE)
s = re.sub(r"^->\s*", "", s)
# take only the first line if the model rambled onto a second
s = s.splitlines()[0].strip() if s.splitlines() else s
# enforce Wallcovering-not-Wallpaper
s = BANNED.sub(lambda m: "Wallcovering" + ("s" if m.group(1) else ""), s)
return s.strip()
PROMPT = (
"You translate Japanese wallcovering PATTERN NAMES into English for a "
"luxury interior-design catalog.\n"
"The names are usually a single material, texture, motif, or design word. "
"MANY are KATAKANA — i.e. a Japanese phonetic spelling of an English, "
"French, or Italian word. In that case recover the ORIGINAL Western word "
"(the material/design term the katakana is spelling), NOT a literal "
"syllable-by-syllable romanization.\n\n"
"Rules:\n"
"- Output ONLY the English name, in Title Case. No quotes, no notes, no "
"romaji, no parentheses, no extra words.\n"
"- Do NOT append 'Wallcovering' or 'Pattern' or any category word — give "
"just the pattern name itself.\n"
"- Do NOT invent or embellish. If it is already English/Latin or a bare "
"product code, return it unchanged.\n"
"- NEVER output the word 'Wallpaper'.\n\n"
"Examples:\n"
"ブークレ -> Bouclé\n"
"ヴァーダァグリィース -> Verdigris\n"
"ラスト -> Rust\n"
"ベロア -> Velour\n"
"スタッコ -> Stucco\n"
"フロステッドグラス -> Frosted Glass\n"
"山ぶどう -> Mountain Grape\n"
"板柾 -> Rift-Sawn Board\n\n"
"{name} ->"
)
def translate_pattern(name, retries=3):
"""Translate a single JA pattern-name segment. Returns English str or None."""
name = name.strip()
if not name:
return ""
# Pure-ASCII (already English / bare code): pass through, just Title Case.
if ASCII_ONLY.match(name):
return title_case(clean_en(name))
body = json.dumps({
"model": MODEL,
"prompt": PROMPT.format(name=name),
"stream": False,
"options": {"temperature": 0, "num_predict": 40},
}).encode("utf-8")
for attempt in range(1, retries + 1):
try:
req = urllib.request.Request(
OLLAMA, data=body,
headers={"Content-Type": "application/json"})
with urllib.request.urlopen(req, timeout=120) as resp:
data = json.loads(resp.read().decode("utf-8"))
raw = data.get("response", "")
en = clean_en(raw)
if not en:
raise ValueError("empty response")
en = title_case(en)
# sanity: reject a response that's absurdly long (model rambled)
if len(en) > 120:
en = en.split(".")[0].strip()[:120]
return en
except Exception as e: # noqa: BLE001 - log + retry/skip
log(f"WARN translate attempt {attempt}/{retries} for "
f"{name!r}: {e}")
time.sleep(1.5 * attempt)
return None
def split_title(title_ja):
"""Return (pattern_ja, sku_range, suffix_ja)."""
parts = [p.strip() for p in title_ja.split("｜")]
if len(parts) >= 3:
return parts[0], parts[1], parts[-1]
if len(parts) == 2:
return parts[0], parts[1], ""
return parts[0], "", ""
def load_done():
done = set()
if os.path.exists(OUT):
with open(OUT, encoding="utf-8") as f:
for line in f:
line = line.strip()
if not line:
continue
try:
done.add(json.loads(line)["sku"])
except Exception: # noqa: BLE001
continue
return done
def main():
done = load_done()
log(f"START translate-titles | source={SRC} | output={OUT} | "
f"already_done={len(done)}")
total = 0
translated = 0
skipped_no_title = 0
skipped_done = 0
errors = 0
with open(SRC, encoding="utf-8") as fin, \
open(OUT, "a", encoding="utf-8") as fout:
for line in fin:
line = line.strip()
if not line:
continue
total += 1
try:
rec = json.loads(line)
except Exception as e: # noqa: BLE001
log(f"ERROR bad json line {total}: {e}")
errors += 1
continue
sku = rec.get("mfr_sku")
title_ja = rec.get("title_ja")
if sku in done:
skipped_done += 1
continue
if not title_ja:
skipped_no_title += 1
continue
pattern_ja, sku_range, suffix_ja = split_title(title_ja)
pattern_en = translate_pattern(pattern_ja)
if pattern_en is None:
log(f"SKIP {sku}: translation failed for pattern "
f"{pattern_ja!r}")
errors += 1
continue
suffix_en = SUFFIX_MAP.get(suffix_ja, "")
if not suffix_en and suffix_ja:
# unknown suffix -> translate it too
s = translate_pattern(suffix_ja)
suffix_en = s or ""
# Reassemble a clean English title.
bits = [pattern_en]
if sku_range:
bits.append(sku_range)
if suffix_en:
bits.append(suffix_en)
title_en = " | ".join(b for b in bits if b)
title_en = BANNED.sub(lambda m: "Wallcovering", title_en)
out_rec = {
"sku": sku,
"title_ja": title_ja,
"title_en": title_en,
"pattern_ja": pattern_ja,
"pattern_en": pattern_en,
"suffix_en": suffix_en,
}
fout.write(json.dumps(out_rec, ensure_ascii=False) + "\n")
fout.flush()
done.add(sku)
translated += 1
if translated % 25 == 0:
log(f"progress: translated={translated} "
f"(scanned {total}) last={sku} -> {title_en!r}")
log(f"DONE translate-titles | scanned={total} translated={translated} "
f"skipped_done={skipped_done} skipped_no_title={skipped_no_title} "
f"errors={errors} | output_total={len(done)}")
if __name__ == "__main__":
main()