`fix_lilycolor_titles.py`

55 lines
#!/usr/bin/env python3
"""
Cleanup pass for enrich_lilycolor_titles.py: any title_en that still contains
Japanese (a failed batch fell back to the raw katakana name) gets re-translated
ONE name at a time — per-item calls can't count-mismatch, so they're reliable.
Atomic temp+rename write. $0 local (qwen2.5vl:7b, already loaded).
"""
import os, json, re, urllib.request

SRC = "staging/lilycolor-unified-staging.jsonl"
OLLAMA = "http://127.0.0.1:11434/api/generate"
MODEL = "qwen2.5vl:7b"
CJK = re.compile(r'[぀-ゟ゠-ヿ一-鿿]')

def translate_one(name):
    prompt = ("Translate this Japanese wallcovering pattern name to a concise English "
              "design term (1-3 words, e.g. Boucle, Velour, Damask, Stucco). "
              "Output ONLY the English term, nothing else.\n" + name)
    body = json.dumps({"model": MODEL, "stream": False, "prompt": prompt,
                       "options": {"temperature": 0, "num_predict": 24}}).encode()
    req = urllib.request.Request(OLLAMA, data=body, headers={"Content-Type": "application/json"})
    raw = json.loads(urllib.request.urlopen(req, timeout=60).read())["response"]
    if "</think>" in raw: raw = raw.split("</think>")[-1]
    # first non-empty line, strip quotes/punctuation
    line = next((l.strip().strip('"\'`.').strip() for l in raw.splitlines() if l.strip()), "")
    return line

def main():
    rows = [json.loads(l) for l in open(SRC) if l.strip()]
    need = [r for r in rows if r.get("title_en") and CJK.search(r["title_en"])]
    uniq = sorted({r["title_en"] for r in need})
    print(f"{len(need)} rows still Japanese across {len(uniq)} unique names", flush=True)
    fixed = {}
    for i, jp in enumerate(uniq):
        try:
            en = translate_one(jp)
            if en and not CJK.search(en): fixed[jp] = en
        except Exception as ex:
            print(f"  '{jp}' failed ({ex})", flush=True)
        if (i + 1) % 25 == 0: print(f"  {i+1}/{len(uniq)}", flush=True)
    n = 0
    for r in rows:
        t = r.get("title_en")
        if t and t in fixed:
            r["title_en"] = fixed[t]; r["title_en_source"] = "qwen-mt-draft-peritem"; n += 1
    TMP = SRC + ".tmp"
    with open(TMP, "w") as f:
        for r in rows: f.write(json.dumps(r, ensure_ascii=False) + "\n")
    os.replace(TMP, SRC)
    remain = sum(1 for r in rows if r.get("title_en") and CJK.search(r["title_en"]))
    print(f"\nDONE: fixed {n} rows; {remain} still Japanese (untranslatable/failed) -> {SRC}", flush=True)

if __name__ == "__main__":
    main()