`scripts/parse_trade.py`

47 lines
#!/usr/bin/env python3
"""Parse trade-portal PLP innerText dumps in data/ -> data/trade_rows.csv (Sanderson only, trade+SSP USD)."""
import re, csv, os
from statistics import mean
DATA = os.path.join(os.path.dirname(__file__), '..', 'data')
CODE = re.compile(r'^([A-Z0-9]{5,12})\s-\s(Wallpaper|Fabric)\s-\s(Live|Discontinued|Limited Stock|New In|New|Coming Soon)\s*$')
PRICE = re.compile(r'^\$([\d,]+(?:\.\d+)?)$')
def num(s): return float(s.replace(',',''))
rows=[]; seen=set()
DROPLOG=os.path.join(DATA,'parse_dropped.log')
with open(DROPLOG,'w'): pass
for cat in ['wallpaper','fabric']:
    fp=os.path.join(DATA,f'sanderson_{cat}_raw.txt')
    if not os.path.exists(fp): continue
    with open(fp) as fh: L=[x.strip() for x in fh.read().split('\n')]
    for i,line in enumerate(L):
        m=CODE.match(line)
        if not m: continue
        code,ptype,status=m.groups()
        prev=[x for x in L[max(0,i-4):i] if x]
        brand=prev[-2] if len(prev)>=2 else ''; name=prev[-1] if prev else ''
        if brand!='Sanderson' or code in seen: continue
        seg=L[i+1:i+12]; trade=ssp=None; unit=''
        for j,x in enumerate(seg):
            if x=='Trade':
                for k in range(j-1,-1,-1):
                    pm=PRICE.match(seg[k])
                    if pm: trade=num(pm.group(1)); break
            if x=='SSP':
                for k in range(j-1,-1,-1):
                    pm=PRICE.match(seg[k])
                    if pm: ssp=num(pm.group(1)); break
            if x.startswith('Per '): unit=x
        if trade is None or ssp is None:
            with open(DROPLOG,'a') as dl: dl.write(f"{cat} {code} {name} - {status} (no price)\n")
            continue
        seen.add(code)
        rows.append({'mfr_code':code,'pattern_name':name,'product_type':ptype,'status':status,
                     'trade_usd':trade,'ssp_usd':ssp,'unit':unit})
with open(os.path.join(DATA,'trade_rows.csv'),'w',newline='') as f:
    w=csv.DictWriter(f,fieldnames=['mfr_code','pattern_name','product_type','status','trade_usd','ssp_usd','unit']); w.writeheader()
    for r in rows: w.writerow(r)
wp=sum(r['product_type']=='Wallpaper' for r in rows); fb=sum(r['product_type']=='Fabric' for r in rows)
ratios=[r['ssp_usd']/r['trade_usd'] for r in rows if r['trade_usd']]
print(f"parsed {len(rows)} Sanderson rows: {wp} wallpaper, {fb} fabric")
if ratios: print(f"SSP/Trade min={min(ratios):.3f} max={max(ratios):.3f} avg={mean(ratios):.3f}")