← back to Japan Enrich
scrape_greenland.py
155 lines
#!/usr/bin/env python3
"""
Greenland Wallcoverings scraper → staging/greenland-staging.jsonl
Greenland = Hangzhou Greenland Import & Export Co. (greenlandwallcoverings.com),
"Global Leading Manufacturer of Natural Textile Wallcoverings" — the vendor the
viewer's china.designerwallcoverings.com host is scoped to (source:'greenland').
The public storefront is a Vue SPA backed by a JSON API at
https://api.greenlandwallcoverings.com/api (no auth needed for reads)
We hit three feed-first endpoints (no headless browser required):
GET /category/list → material tree (Sisal/Hemp/…)
GET /product/page?categoryId=<id>&pageSize=… → PATTERN records per category
GET /product/search?pageSize=… → COLORWAY records (model = SKU)
A pattern (productId) explodes to N colorway SKUs — exactly the Sangetsu shape.
We emit ONE staging row per pattern, colorways nested, so the viewer's buildRows
greenland branch can fan each colorway to a card (title = pattern, image = per
colorway thumbnail). Images are served remotely via /api/downloadFile and render
fine as external URLs, so no local image fetch on this first pass.
"""
import json, sys, time, urllib.request, urllib.error, os
API = "https://api.greenlandwallcoverings.com/api"
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) japan-enrich/greenland"
OUT = os.path.join(os.path.dirname(__file__), "staging", "greenland-staging.jsonl")
def get(path, tries=4):
url = API + path
for i in range(tries):
try:
req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "application/json"})
with urllib.request.urlopen(req, timeout=60) as r:
return json.loads(r.read().decode("utf-8"))
except (urllib.error.URLError, TimeoutError, json.JSONDecodeError) as e:
if i == tries - 1:
print(f" ! give up {path}: {e}", file=sys.stderr)
return None
time.sleep(1.5 * (i + 1))
return None
# Branch classification by ANCESTOR id (not top-level root — Category(4) and
# Collections(5) both sit under Wallcovering(1), so roots can't separate them).
# Fiber taxonomy lives under these subtrees → real MATERIAL. "Collections"(5) and
# "What's New"(6) are books/campaigns (Story of Nature, Wabi Sabi…) → not a fiber.
MATERIAL_ANCESTORS = {2, 3, 4} # Printing Substrate, Engineering wallpaper, Wallcovering>Category
def flatten_tree(nodes, anc=None, out=None):
"""category/list is a nested tree; return {id: {label, path[], is_leaf, anc:set}}.
`anc` = set of ancestor ids INCLUDING self, so a category can be tested for
membership in the material subtree regardless of depth."""
if out is None:
out = {}
anc = anc or ()
for n in nodes or []:
cid = n.get("id")
label = (n.get("label") or "").strip()
chain = set(anc) | {cid}
kids = n.get("children") or []
out[cid] = {"label": label, "is_leaf": not kids, "anc": chain}
if kids:
flatten_tree(kids, chain, out)
return out
def main():
print("[greenland] category tree…")
cat = get("/category/list")
tree = flatten_tree(cat["data"]) if cat and cat.get("code") == 0 else {}
print(f" {len(tree)} category nodes")
# productId -> deepest (most specific) material label, by paging each category.
# Deeper path wins so a pattern lands on "Sisal" not just "Wallcovering".
material = {} # productId -> {"label":..., "depth":int} (fiber only, deepest wins)
pat_meta = {} # productId -> {name, code, coverImg, categoryName}
for cid, meta in sorted(tree.items()):
is_material = bool(meta["anc"] & MATERIAL_ANCESTORS)
depth = len(meta["anc"])
d = get(f"/product/page?categoryId={cid}&page=1&pageSize=3000")
rows = (d or {}).get("data", {}).get("list") or []
if not rows:
continue
for r in rows:
pid = r.get("ID")
if pid is None:
continue
pat_meta.setdefault(pid, {
"name": (r.get("name") or "").strip(),
"code": (r.get("code") or "").strip(),
"coverImg": r.get("coverImg") or None,
"categoryName": (r.get("categoryName") or "").strip() or None,
})
# only fiber-taxonomy categories set material; skip Collections/What's-New/color leaves
if is_material and meta["label"]:
cur = material.get(pid)
if cur is None or depth > cur["depth"]:
material[pid] = {"label": meta["label"], "depth": depth}
print(f" cat {cid:>3} {'M' if is_material else ' '} {meta['label'][:18]:<18} +{len(rows)}")
# All colorways in one shot, grouped by productId.
print("[greenland] colorways (/product/search)…")
d = get("/product/search?page=1&pageSize=6000")
cws = (d or {}).get("data", {}).get("list") or []
print(f" {len(cws)} colorways")
groups = {}
for c in cws:
groups.setdefault(c.get("productId"), []).append(c)
# Emit one staging row per pattern. Union of patterns seen in either feed.
pids = set(pat_meta) | set(groups)
os.makedirs(os.path.dirname(OUT), exist_ok=True)
n = 0
with open(OUT, "w") as f:
for pid in sorted(x for x in pids if x is not None):
cw = groups.get(pid, [])
meta = pat_meta.get(pid, {})
# pattern name: prefer /product/page name, else colorway productName
pattern = meta.get("name") or (cw[0].get("productName").strip() if cw and cw[0].get("productName") else f"Greenland {pid}")
skus, imgs, colors = [], {}, {}
for c in cw:
sku = (c.get("model") or "").strip() or (c.get("code") or "").strip()
if not sku or sku in imgs:
continue
skus.append(sku)
if c.get("thumbnail"):
imgs[sku] = c["thumbnail"]
cname = (c.get("colorDescription") or c.get("colorName") or "").strip()
if cname:
colors[sku] = cname
row = {
"product_id": pid,
"pattern": pattern,
"mfr_prefix": (meta.get("code") or "")[:1] or None,
"product_code": meta.get("code") or None,
"category": meta.get("categoryName"), # top ("Wallcovering")
"material": (material.get(pid) or {}).get("label"), # leaf ("Sisal") → facet
"colorway_skus": skus,
"sku_images": imgs,
"sku_color": colors,
"cover_img": meta.get("coverImg"),
"source_url": f"https://www.greenlandwallcoverings.com/detail?id={pid}",
}
f.write(json.dumps(row, ensure_ascii=False) + "\n")
n += 1
tot_cw = sum(len(g) for g in groups.values())
print(f"[greenland] wrote {n} patterns / {tot_cw} colorways → {OUT}")
if __name__ == "__main__":
main()