`enrich_lilycolor_titles.py`

92 lines
#!/usr/bin/env python3
"""
Lilycolor title_en synthesizer — DRAFT English titles ($0 local, qwen3:14b).

title_ja is structured "PatternName｜SKU-range｜suffix" and the pattern name is
mostly katakana loanwords (ブークレ=Bouclé, ベロア=Velour). We:
  1) extract the pattern name = title_ja.split('｜')[0]
  2) batch-translate the UNIQUE set of names via qwen3:14b (local, free)
  3) rows with no title_ja get a derived "Lilycolor <Collection>" label
Every synthesized title is marked title_en_source so it's clearly a DRAFT —
these are activation-gated and NOT customer-live until Steve reviews.

Atomic temp+rename write. Idempotent: skips rows that already have title_en.
"""
import os, json, urllib.request

SRC = "staging/lilycolor-unified-staging.jsonl"
OLLAMA = "http://127.0.0.1:11434/api/generate"
MODEL = "qwen2.5vl:7b"   # already-loaded; avoids OLLAMA_MAX_LOADED swap + qwen3 <think> stall
PIPE = "｜"  # U+FF5C fullwidth pipe

CATALOG_LABEL = {
    "LIGHT": "Light", "WILL": "Will", "V-wall": "V-Wall",
    "MATERIALS": "Materials", "ImportSelection": "Import Selection",
}

def qwen(prompt):
    body = json.dumps({"model": MODEL, "stream": False, "prompt": prompt,
                       "options": {"temperature": 0}}).encode()
    req = urllib.request.Request(OLLAMA, data=body, headers={"Content-Type": "application/json"})
    return json.loads(urllib.request.urlopen(req, timeout=180).read())["response"]

def translate_batch(names):
    numbered = "\n".join(f"{i+1}. {n}" for i, n in enumerate(names))
    prompt = ("Translate these Japanese wallcovering pattern names to concise English "
              "design terms (1-3 words each; keep proper design vocabulary like Boucle, "
              "Velour, Damask). Output ONLY a JSON array of strings in the same order, "
              "no commentary, no <think>.\n" + numbered)
    raw = qwen(prompt)
    # strip any <think>…</think> and grab the JSON array
    if "</think>" in raw: raw = raw.split("</think>")[-1]
    s, e = raw.find("["), raw.rfind("]")
    arr = json.loads(raw[s:e+1])
    if len(arr) != len(names):
        raise ValueError(f"count mismatch {len(arr)} != {len(names)}")
    return [str(x).strip() for x in arr]

def main():
    rows = [json.loads(l) for l in open(SRC) if l.strip()]
    # unique katakana pattern names to translate
    def pname(r):
        t = r.get("title_ja")
        return t.split(PIPE)[0].strip() if t else None
    uniq = sorted({pname(r) for r in rows if pname(r)})
    print(f"{len(rows)} rows | {len(uniq)} unique JP pattern names to translate", flush=True)

    mapping = {}
    B = 25
    for i in range(0, len(uniq), B):
        chunk = uniq[i:i+B]
        try:
            en = translate_batch(chunk)
        except Exception as ex:
            print(f"  batch {i//B} retry ({ex})", flush=True)
            try: en = translate_batch(chunk)
            except Exception as ex2:
                print(f"  batch {i//B} FAILED, romaji-fallback ({ex2})", flush=True)
                en = chunk  # last resort: keep original (better than blank)
        for jp, e in zip(chunk, en): mapping[jp] = e
        print(f"  translated {min(i+B,len(uniq))}/{len(uniq)}", flush=True)

    n_mt = n_derived = n_skip = 0
    for r in rows:
        if r.get("title_en"):  # idempotent
            n_skip += 1; continue
        p = pname(r)
        if p and mapping.get(p):
            r["title_en"] = mapping[p]; r["title_en_source"] = "qwen3-mt-draft"; n_mt += 1
        else:
            lbl = CATALOG_LABEL.get(r.get("source_catalog"), r.get("source_catalog") or "Lilycolor")
            r["title_en"] = f"Lilycolor {lbl} — {r.get('mfr_sku')}"
            r["title_en_source"] = "derived"; n_derived += 1

    TMP = SRC + ".tmp"
    with open(TMP, "w") as f:
        for r in rows: f.write(json.dumps(r, ensure_ascii=False) + "\n")
    os.replace(TMP, SRC)
    print(f"\nDONE: title_en set on {n_mt} (mt-draft) + {n_derived} (derived); {n_skip} already had one -> {SRC}", flush=True)

if __name__ == "__main__":
    main()