`pull.py`

154 lines
#!/usr/bin/env python3
"""Pull Zendesk Chat history via the REST API and emit public/data.json.
Token is read server-side from secrets-manager/.env and NEVER written to output."""
import json, os, re, urllib.request, urllib.parse, sys, time

ENV = os.path.expanduser("~/Projects/secrets-manager/.env")
OUT = os.path.join(os.path.dirname(__file__), "public", "data.json")
BASE = "https://www.zopim.com/api/v2/incremental/chats"
START = int(os.environ.get("START", "1760000000"))   # wide window to capture full history
MAX_PAGES = int(os.environ.get("MAX_PAGES", "80"))

# Theme classifier — regex per theme, matched against visitor text
THEMES = {
    "discontinued/availability": r"discontinu|no longer|out of (stock|production)|still (available|make|carry|sell|produc|in production)|been discontinued|still (get|order|sell)|hard to find|can.?t find|in stock|backorder|availab",
    "order status/shipping": r"where.?s my order|trackin|has.?n.?t (ship|arriv|come)|not (yet )?(received|arrived|delivered)|order status|shipping (info|status|update)|when will .* (ship|arrive|deliver)|still waiting|lead time|how long",
    "samples": r"\bsample|swatch|memo\b|cutting|piece of",
    "trade/pro account": r"trade account|designer discount|to the trade|pro(fessional)? account|interior designer|net price|trade price|resale|reseller",
    "pricing/quote": r"how much|price|cost per|quote|\$|per (roll|yard|yd)|pricing|afford",
    "installation/how-to": r"install|how (do|to) (i )?(apply|hang|paste)|paste the wall|square (feet|footage)|how many rolls|coverage|repeat|match",
    "complaint/issue": r"wrong|damag|defect|broken|torn|missing|refund|return|cancel|terrible|awful|disappoint|bad lot|no (one|response|reply|answer)|rude|charged twice",
}
# Vendor/brand mentions (wallcovering + fabric houses DW carries / gets asked about)
BRANDS = ["Kravet","Thibaut","Schumacher","Lee Jofa","Cole & Son","Phillip Jeffries","Phillipe Romano",
          "Waverly","Brunschwig","Clarke & Clarke","Romo","Scalamandre","Osborne","Zoffany","Designers Guild",
          "Ralph Lauren","York","Hollywood","Avant Garde","GP & J Baker","Groundworks","Pierre Frey","Arte",
          "Maya Romanoff","Koroseal","Wolf Gordon","Gaston","Farrow","Sanderson","Morris","Rebel Walls","Astek"]


def token():
    if os.environ.get("ZENDESK_CHAT_ACCESS_TOKEN"):
        return os.environ["ZENDESK_CHAT_ACCESS_TOKEN"].strip()
    try:
        for line in open(ENV):
            if line.startswith("ZENDESK_CHAT_ACCESS_TOKEN="):
                return line.split("=", 1)[1].strip().strip('"').strip("'")
    except Exception:
        pass
    raise SystemExit("no token")


def fetch(url, tok):
    req = urllib.request.Request(url, headers={"Authorization": "Bearer " + tok})
    with urllib.request.urlopen(req, timeout=45) as r:
        return json.load(r)


def clean_title(t):
    if not t:
        return ""
    # strip the boilerplate site suffix from product page titles
    t = re.split(r"[–\-|]\s*Designer Wallcoverings", t)[0]
    t = re.sub(r"\s+", " ", t).strip()
    return t


def classify(text):
    t = (text or "").lower()
    return [name for name, rx in THEMES.items() if re.search(rx, t)]


def find_brands(text):
    t = (text or "").lower()
    return sorted({b for b in BRANDS if b.lower() in t})


NEG = r"wrong|damag|defect|broken|torn|missing|refund|return|cancel|terrible|awful|horrible|disappoint|frustrat|angry|upset|unhappy|not happy|problem|issue|never (received|got|arrived)|still waiting|ridiculous|unacceptable|complain|poor|bad (lot|quality|experience)|worst|rude|ignored|no (one|response|reply|answer)"
POS = r"love|great|beautiful|perfect|thank you|thanks|excellent|wonderful|happy|gorgeous|amazing|fantastic|pleased|appreciate|awesome|excited|can.?t wait|lovely|stunning"
URG = r"\basap\b|urgent|immediately|right away|deadline|need (it |them )?by|running out|time.?sensitive|today|tomorrow|rush|as soon as|quickly|expedite|in a hurry|last minute"


def sentiment(text):
    t = (text or "").lower()
    if not t.strip():
        return ""
    neg = len(re.findall(NEG, t)); pos = len(re.findall(POS, t))
    if neg > pos:
        return "negative"
    if pos > neg:
        return "positive"
    return "neutral"


def urgency(text):
    return "high" if re.search(URG, (text or "").lower()) else "normal"


def main():
    tok = token()
    url = BASE + "?" + urllib.parse.urlencode({"fields": "chats(*)", "start_time": START})
    rows, pages = [], 0
    while url and pages < MAX_PAGES:
        d = fetch(url, tok)
        pages += 1
        chats = d.get("chats", [])
        for c in chats:
            s = c.get("session") or {}
            hist = c.get("history") or []
            msgs = [h.get("msg", "") for h in hist if isinstance(h, dict) and h.get("msg")]
            # visitor msgs: sender_type == 'visitor' (API lowercases it)
            vmsgs = [h.get("msg", "") for h in hist
                     if isinstance(h, dict) and h.get("msg") and str(h.get("sender_type", "")).lower() == "visitor"]
            if not vmsgs and c.get("comment"):
                vmsgs = [c.get("comment")]   # offline messages carry text in comment
            titles = [clean_title(w.get("title")) for w in (c.get("webpath") or []) if w.get("title")]
            vtext = " ".join(vmsgs)
            themes = classify(vtext)
            brands = find_brands(vtext + " " + " ".join(titles) + " " + " ".join(c.get("tags") or []))
            rows.append({
                "themes": themes,
                "brands": brands,
                "sentiment": sentiment(vtext),
                "urgency": urgency(vtext),
                "id": c.get("id"),
                "ts": c.get("timestamp"),
                "type": c.get("type"),
                "country_code": s.get("country_code") or "??",
                "country": s.get("country_name") or "Unknown",
                "city": s.get("city") or "",
                "region": s.get("region") or "",
                "platform": s.get("platform") or "",
                "browser": s.get("browser") or "",
                "agents": c.get("agent_names") or [],
                "duration": c.get("duration") or 0,
                "rating": c.get("rating") or "",
                "comment": c.get("comment") or "",
                "tags": c.get("tags") or [],
                "search_terms": c.get("referrer_search_terms") or "",
                "missed": bool(c.get("missed")),
                "proactive": bool(c.get("proactive")),
                "msg_count": len(msgs),
                "visitor_msgs": vmsgs if vmsgs else msgs,  # fallback to all if role missing
                "titles": titles,
                "ticket_id": c.get("zendesk_ticket_id") or "",
            })
        end = d.get("end_time")
        nxt = d.get("next_page")
        if not chats or not end:
            break
        url = nxt if nxt else BASE + "?" + urllib.parse.urlencode({"fields": "chats(*)", "start_time": end})

    payload = {
        "generated_at": int(time.time()),
        "count": len(rows),
        "pages": pages,
        "chats": rows,
    }
    os.makedirs(os.path.dirname(OUT), exist_ok=True)
    json.dump(payload, open(OUT, "w"), separators=(",", ":"))
    print("wrote %d chats (%d pages) -> %s" % (len(rows), pages, OUT))


if __name__ == "__main__":
    main()