← back to Designerwallcoverings
scripts/artmura-onboard/onboard.py
242 lines
#!/usr/bin/env python3
"""
Artmura full-monte onboarder (feed-first).
Source: https://newwall.com/collections/artmura (Shopify store; Artmura is one brand newwall resells)
Captures: SKU, pattern series, color, collection book, newwall RETAIL price, sample price,
per-PDP spec table (dimensions/width, grade, coverage, lead-time, origin), body_html,
product URL, every product image (downloaded locally).
PRICING NOTE (hard rule): the $235-360/yd prices are newwall.com's RETAIL, NOT our wholesale cost.
This script captures them as `price_newwall_retail` only. No DW sell price is computed and no Shopify
push happens here — that is gated on Artmura cost + trade discount (see scraper-must-confirm-cost rule).
Outputs:
data/artmura.json structured master package (161 products)
images/<sku>-<n>.jpg every product image
DB refresh is done by db_write.py (reads data/artmura.json — no re-scrape, so it can't
clobber filled specs). Run order: onboard.py -> fill_specs.py -> db_write.py.
"""
import json, os, re, sys, time, html, urllib.request, concurrent.futures as cf
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
OUT_DIR = os.path.join(HERE, "data")
IMG_DIR = os.path.join(ROOT, ".newwall-ref", "artmura", "images")
os.makedirs(OUT_DIR, exist_ok=True)
os.makedirs(IMG_DIR, exist_ok=True)
COLLECTION_URL = "https://newwall.com/collections/artmura/products.json?limit=250"
PDP_BASE = "https://newwall.com/products/"
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
# tags that are NOT the pattern series (generic / merchandising / collection-book labels)
NON_SERIES = {"Artmura", "maria picks", "Trending", "Contemporary"}
BOOK_TAGS = {"Opera", "La Scala Milano"} # parent collection "books"
def fetch(url, tries=3, binary=False):
last = None
for i in range(tries):
try:
req = urllib.request.Request(url, headers={"User-Agent": UA})
with urllib.request.urlopen(req, timeout=30) as r:
return r.read() if binary else r.read().decode("utf-8", "replace")
except Exception as e:
last = e; time.sleep(1.5 * (i + 1))
raise last
def parse_specs(pdp_html):
pairs = re.findall(
r'product-info__label">([^<]+)</th>\s*<td class="product-info__value">(.*?)</td>',
pdp_html, re.S)
specs, seen = {}, set()
for lbl, val in pairs:
lbl = lbl.strip().rstrip(":")
if lbl in seen:
continue
seen.add(lbl)
specs[lbl] = re.sub(r"<[^>]+>", "", val).strip()
return specs
def width_inches(dim):
if not dim:
return None
m = re.search(r'([\d.]+)\s*"', dim) or re.search(r'([\d.]+)\s*(?:in|inch)', dim)
return float(m.group(1)) if m else None
def main():
no_img = "--no-images" in sys.argv
if "--write-db" in sys.argv:
sys.exit("DB refresh moved to db_write.py (reads the filled package without re-scraping). "
"Run: python3 db_write.py")
print("→ fetching collection feed ...")
feed = json.loads(fetch(COLLECTION_URL))["products"]
print(f" {len(feed)} products")
# 1) fetch all PDP spec tables concurrently
print("→ fetching per-product spec tables ...")
specs_by_handle = {}
def grab(p):
try:
h = fetch(PDP_BASE + p["handle"])
return p["handle"], parse_specs(h)
except Exception as e:
return p["handle"], {"_error": str(e)}
with cf.ThreadPoolExecutor(max_workers=6) as ex:
for handle, specs in ex.map(grab, feed):
specs_by_handle[handle] = specs
ok = sum(1 for s in specs_by_handle.values() if s and "_error" not in s)
print(f" specs parsed for {ok}/{len(feed)}")
# 2) build records
records = []
for p in feed:
title = html.unescape(p["title"]).strip()
tags = p.get("tags", [])
series = next((t for t in tags if t not in NON_SERIES and t not in BOOK_TAGS), None) \
or next((t for t in tags if t not in NON_SERIES), None) or title.split()[0]
book = next((t for t in tags if t in BOOK_TAGS), None)
color = title[len(series):].strip() if title.lower().startswith(series.lower()) else title
yard = next((v for v in p["variants"] if (v["option1"] or "").lower() != "sample"), p["variants"][0])
sample = next((v for v in p["variants"] if (v["option1"] or "").lower() == "sample"), None)
sku = (yard.get("sku") or "").strip().upper()
specs = specs_by_handle.get(p["handle"], {})
dim = specs.get("Dimensions")
imgs = [im["src"].split("?")[0] for im in p.get("images", [])]
records.append({
"mfr_sku": sku,
"vendor": "Artmura",
"pattern_series": series,
"color": color or None,
"title": title,
"collection_book": book,
"tags": tags,
"product_type": p.get("product_type") or "Wallcoverings",
"price_newwall_retail": float(yard["price"]), # newwall RETAIL, NOT our cost
"sample_price": float(sample["price"]) if sample else None,
"sold_by": yard.get("option1") or "Yard",
"substrate": yard.get("option2") or specs.get("Substrate available"),
"grade": specs.get("Grade available"),
"dimensions": dim,
"width_inches": width_inches(dim),
"wall_coverage": specs.get("Wall coverage"),
"lead_time": specs.get("Lead-time"),
"origin": specs.get("Origin"),
"body_html": p.get("body_html"),
"product_url": PDP_BASE + p["handle"],
"handle": p["handle"],
"shopify_source_id": p["id"],
"images": imgs,
"image_count": len(imgs),
"published_at": p.get("published_at"),
})
# 3) download images
if not no_img:
print("→ downloading images ...")
jobs = []
for r in records:
for i, src in enumerate(r["images"], 1):
ext = os.path.splitext(src.split("/")[-1])[1] or ".jpg"
fn = f"{r['mfr_sku']}-{i}{ext}"
jobs.append((src, os.path.join(IMG_DIR, fn), fn))
done = [0]; fails = []
def dl(job):
src, path, fn = job
if os.path.exists(path) and os.path.getsize(path) > 0:
return fn, True
try:
data = fetch(src, binary=True)
with open(path, "wb") as f:
f.write(data)
return fn, True
except Exception as e:
return fn, str(e)
with cf.ThreadPoolExecutor(max_workers=10) as ex:
for fn, res in ex.map(dl, jobs):
done[0] += 1
if res is not True:
fails.append((fn, res))
print(f" {done[0]-len(fails)}/{len(jobs)} images on disk ({len(fails)} failed)")
for r in records:
r["local_images"] = [
f".newwall-ref/artmura/images/{r['mfr_sku']}-{i}{os.path.splitext(s.split('/')[-1])[1] or '.jpg'}"
for i, s in enumerate(r["images"], 1)]
# 4) write master package
pkg_path = os.path.join(OUT_DIR, "artmura.json")
with open(pkg_path, "w") as f:
json.dump({
"source": "https://newwall.com/collections/artmura",
"vendor": "Artmura",
"captured_count": len(records),
"price_note": "price_newwall_retail = newwall.com retail; NOT DW wholesale cost. Shopify push gated on Artmura cost + trade discount.",
"products": records,
}, f, indent=2)
print(f"→ wrote {pkg_path}")
# summary
series_ct = {}
for r in records:
series_ct[r["pattern_series"]] = series_ct.get(r["pattern_series"], 0) + 1
print("\n=== CAPTURE SUMMARY ===")
print(f"products : {len(records)}")
print(f"images : {sum(r['image_count'] for r in records)}")
print(f"w/ color : {sum(1 for r in records if r['color'])}")
print(f"w/ dimensions : {sum(1 for r in records if r['dimensions'])}")
print(f"w/ origin : {sum(1 for r in records if r['origin'])}")
print(f"price range : ${min(r['price_newwall_retail'] for r in records):.0f}-${max(r['price_newwall_retail'] for r in records):.0f}/yd (newwall retail)")
print(f"series : {len(series_ct)} -> {dict(sorted(series_ct.items(), key=lambda x:-x[1])[:8])}")
print("\n(next: python3 fill_specs.py then python3 db_write.py)")
def _unused_refresh_db(records):
import subprocess
cs = open("/tmp/.dwcs").read().strip()
print("\n→ refreshing newwall_catalog rows (by handle) ...")
updated = 0
for r in records:
all_images = "{" + ",".join('"%s"' % s for s in r["images"]) + "}"
# store ONLY newwall retail as price_retail reference; leave price_trade/cost untouched-null
sql = """
UPDATE newwall_catalog SET
mfr_sku=%(sku)s, pattern_name=%(pat)s, color_name=%(col)s,
collection=%(book)s, product_type=%(ptype)s, material=%(sub)s,
width=%(width)s, coverage=%(cov)s, price_retail=%(retail)s,
image_url=%(img)s, all_images=%(allimg)s, body_html=%(body)s,
us_distributor='newwall.com (reseller)', about_vendor=%(about)s,
last_scraped=now(), updated_at=now()
WHERE handle=%(handle)s
"""
params = {
"sku": r["mfr_sku"], "pat": r["pattern_series"], "col": r["color"],
"book": r["collection_book"], "ptype": r["product_type"],
"sub": r["substrate"], "width": r["dimensions"],
"cov": r["wall_coverage"], "retail": r["price_newwall_retail"],
"img": r["images"][0] if r["images"] else None, "allimg": all_images,
"body": r["body_html"],
"about": f"Artmura — Italian non-woven wallcovering. Grade: {r.get('grade')}. Origin: {r.get('origin')}. Lead-time: {r.get('lead_time')}.",
"handle": r["handle"],
}
# use psql with -v params via a here-doc through python -> safer: build with psycopg-less escaping
def esc(v):
if v is None: return "NULL"
if isinstance(v,(int,float)): return str(v)
return "'" + str(v).replace("'", "''") + "'"
q = f"""UPDATE newwall_catalog SET
mfr_sku={esc(params['sku'])}, pattern_name={esc(params['pat'])}, color_name={esc(params['col'])},
collection={esc(params['book'])}, product_type={esc(params['ptype'])}, material={esc(params['sub'])},
width={esc(params['width'])}, coverage={esc(params['cov'])}, price_retail={esc(params['retail'])},
image_url={esc(params['img'])}, all_images={esc(params['allimg'])}, body_html={esc(params['body'])},
us_distributor='newwall.com (reseller)', about_vendor={esc(params['about'])},
last_scraped=now(), updated_at=now()
WHERE handle={esc(params['handle'])};"""
res = subprocess.run(["psql", cs, "-tAc", q], capture_output=True, text=True)
if res.returncode == 0 and "UPDATE 1" in res.stdout:
updated += 1
elif res.stderr.strip():
print(" ! ", r["mfr_sku"], res.stderr.strip()[:120])
print(f" updated {updated}/{len(records)} rows")
if __name__ == "__main__":
main()