`tk11041-innovations-reconcile/rescrape/extract.py`

71 lines
import re,html,json,os,csv
BASE='/private/tmp/claude-501/-Users-macstudio3-Projects-designerwallcoverings/92987ed5-7379-4db6-b4d6-853407d8799b/scratchpad/inn'
MAP={'CSO':('costine','Costine','live'),'ENS':('enchase','Enchase','live'),
     'FLN':('flanders','Flanders','live'),'PRO':('province','Province','live'),
     'SKL':('skylark','Skylark','live'),'TTM':('tatami','Tatami','live'),
     'VCO':('vinyl cork','Vinyl Cork','live'),'WHL':('whistler','Whistler','discontinued')}
FILES={'CSO':'costine','ENS':'enchase','FLN':'flanders','PRO':'province','SKL':'skylark',
       'TTM':'tatami','VCO':'vinyl_cork','WHL':'_wayback_whistler'}
SPEC_KEYS=['Composition','Backing','Total Weight','Width','Repeat','Full Roll','Fire Rating','Seaming','Environmental Impact and Sustainability']

def flat(p):
    h=open(p,encoding='utf-8',errors='replace').read()
    t=re.sub(r'<(script|style)[^>]*>.*?</\1>',' ',h,flags=re.S|re.I)
    return html.unescape(re.sub(r'\s+',' ',re.sub(r'<[^>]+>',' ',t))), h

def specs(t):
    out={}
    i=t.find('Specifications')
    if i<0: return out
    seg=t[i:i+900]
    for k in SPEC_KEYS:
        m=re.search(re.escape(k)+r':\s*(.+?)(?=\s(?:'+'|'.join(re.escape(x) for x in SPEC_KEYS)+r'):|\s*Product Downloads|\s*Seen In|$)',seg)
        if m: out[k]=m.group(1).strip()
    return out

def colorways(t,pfx):
    d={}
    for m in re.finditer(r'\b('+pfx+r'-\d{2,3})\s+([A-Z][A-Za-z ]{1,24}?)(?=\s+'+pfx+r'-\d|\s+Item is|\s+ORDER|\s+Select|\s+Line \d|\s{2,}|$)',t):
        code,name=m.group(1),m.group(2).strip()
        if name.lower() in ('item','order','select','line'): continue
        d.setdefault(code,name.title())
    return d

pat={}
for pfx,fn in FILES.items():
    p=os.path.join(BASE,'pages',fn+'.html')
    t,h=flat(p)
    slug,disp,state=MAP[pfx]
    pat[pfx]={'slug':slug,'pattern':disp,'site_state':state,'specs':specs(t),
              'colorways':colorways(t,pfx),
              'desc':(re.search(r'"description":"(.*?)"',h).group(1)[:400] if re.search(r'"description":"(.*?)"',h) else ''),
              'product_url':('https://www.innovationsusa.com/item/'+slug.replace(' ','%20')) if state=='live' else 'DISCONTINUED (archived: web.archive.org/web/20230127073736/https://www.innovationsusa.com/item/whistler)'}

rows=[]
for line in open('/tmp/cohort51.tsv',encoding='utf-8'):
    line=line.rstrip('\n')
    if not line: continue
    mfr,title=line.split('\t',1)
    pfx=mfr.split('-')[0]
    P=pat.get(pfx,{})
    cw=P.get('colorways',{}).get(mfr,'')
    dwcw=re.search(r'\b([A-Z][A-Z \-]{2,})\s+Memo Sample',title)
    rows.append({'mfr_sku':mfr,'dw_title':title,'innovations_pattern':P.get('pattern',''),
        'innovations_colorway':cw,'dw_colorway':(dwcw.group(1).strip().title() if dwcw else ''),
        'colorway_match':('YES' if cw and dwcw and cw.lower()==dwcw.group(1).strip().lower() else ('n/a' if not cw else 'CHECK')),
        'site_state':P.get('site_state',''),'product_url':P.get('product_url',''),
        'image_url':f'https://www.innovationsusa.com/storage/sku/350x350/{mfr}.jpg' if P.get('site_state')=='live' else '',
        'width':P.get('specs',{}).get('Width',''),'full_roll':P.get('specs',{}).get('Full Roll',''),
        'composition':P.get('specs',{}).get('Composition',''),'repeat':P.get('specs',{}).get('Repeat',''),
        'fire_rating':P.get('specs',{}).get('Fire Rating','')})

os.makedirs(BASE+'/out',exist_ok=True)
json.dump({'patterns':pat,'items':rows},open(BASE+'/out/innovations-identity-map.json','w'),indent=2)
with open(BASE+'/out/innovations-identity-map.csv','w',newline='') as f:
    w=csv.DictWriter(f,fieldnames=list(rows[0].keys())); w.writeheader(); w.writerows(rows)
res=sum(1 for r in rows if r['innovations_pattern'])
cw=sum(1 for r in rows if r['colorway_match']=='YES')
print(f'items={len(rows)} identity_resolved={res} colorway_confirmed={cw}')
for p,v in pat.items():
    print(f"  {p} -> {v['pattern']:<12} {v['site_state']:<13} colorways={len(v['colorways'])} specs={len(v['specs'])}")