← back to Newmor Onboard
chore: lint, refactor (session close, TK-11424)
193bb47450c8c6d42f19ab7e75a2c78db6b9a104 · 2026-09-13 01:23:15 -0700 · Steve Abrams
Files touched
M scripts/backfill-metafields.mjsM scripts/spec-pdf-refresh.py
Diff
commit 193bb47450c8c6d42f19ab7e75a2c78db6b9a104
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sun Sep 13 01:23:15 2026 -0700
chore: lint, refactor (session close, TK-11424)
---
scripts/backfill-metafields.mjs | 4 +--
scripts/spec-pdf-refresh.py | 58 ++++++++++++++++++++++++-----------------
2 files changed, 36 insertions(+), 26 deletions(-)
diff --git a/scripts/backfill-metafields.mjs b/scripts/backfill-metafields.mjs
index 2925735..1fd568d 100644
--- a/scripts/backfill-metafields.mjs
+++ b/scripts/backfill-metafields.mjs
@@ -92,7 +92,7 @@ const jsonl = []; // one line per proposed write
const restoreMap = []; // reversibility record
const keyTotals = {}; // key -> proposed write count
const skipReasons = {}; // key -> {live_populated, source_empty, would_fork_type}
-for (const cand of CANDIDATES) skipReasons[cand.key] = { live_populated: 0, source_empty: 0 };
+for (const cand of CANDIDATES) skipReasons[cand.key] = { live_populated: 0, source_empty: 0, unparseable_source: 0 };
// First pass: discover existing live type per candidate key across all 116 (to lock type, never fork)
const liveByGid = new Map();
@@ -132,7 +132,7 @@ for (const {mfr, gid} of csvRows) {
// FILL: live empty/absent AND source non-empty
const type = lockedType[cand.key];
const value = cand.format ? cand.format(src) : String(src).trim();
- if (value === null) { rec.skips.push({key: cand.key, reason: 'unparseable_source', src}); skipReasons[cand.key].source_empty++; continue; }
+ if (value === null) { rec.skips.push({key: cand.key, reason: 'unparseable_source', src}); skipReasons[cand.key].unparseable_source++; continue; }
const write = { key: `${NS}.${cand.key}`, type, value, origin: cand.origin, existingMetafieldId: liveM ? liveM.id : null, action: liveM ? 'update-empty' : 'create' };
rec.writes.push(write);
keyTotals[cand.key] = (keyTotals[cand.key] || 0) + 1;
diff --git a/scripts/spec-pdf-refresh.py b/scripts/spec-pdf-refresh.py
old mode 100644
new mode 100755
index 78afecb..8e7f444
--- a/scripts/spec-pdf-refresh.py
+++ b/scripts/spec-pdf-refresh.py
@@ -32,7 +32,17 @@ Usage:
python3 spec-pdf-refresh.py --apply # fill-only UPDATE newmor_catalog (snapshots first)
python3 spec-pdf-refresh.py --all # scrape + stage + apply
"""
-import re, sys, json, time, html, hashlib, subprocess, os, argparse, urllib.request, urllib.error
+import argparse
+import hashlib
+import html
+import json
+import os
+import re
+import subprocess
+import sys
+import time
+import urllib.error
+import urllib.request
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.join(HERE, "..")
@@ -65,15 +75,15 @@ def clean(s):
TECH_RE = re.compile(
r'technical-container__title">\s*(.*?)\s*</div>\s*'
- r'<(a|p)([^>]*)class="technical-container__link[^"]*"([^>]*)>\s*(.*?)\s*</\2>', re.S)
+ r'<(a|p)([^>]*)class="technical-container__link[^"]*"([^>]*)>\s*(.*?)\s*</\2>', re.DOTALL)
def parse_page(url, h):
pat = ""
- m = re.search(r"<h1[^>]*>(.*?)</h1>", h, re.S)
+ m = re.search(r"<h1[^>]*>(.*?)</h1>", h, re.DOTALL)
if m:
pat = clean(m.group(1))
if not pat:
- m = re.search(r"<title>(.*?)</title>", h, re.S)
+ m = re.search(r"<title>(.*?)</title>", h, re.DOTALL)
if m:
pat = clean(m.group(1)).split(" - Newmor")[0].split("|")[0].strip()
@@ -93,8 +103,8 @@ def parse_page(url, h):
for t in tech_items:
title_l = t["title"].lower()
if "roll siz" in title_l or "width" in title_l:
- mm = re.findall(r"(\d{3,4})\s*mm", t["text"], re.I)
- cm = re.findall(r"(\d{2,4})\s*cm", t["text"], re.I)
+ mm = re.findall(r"(\d{3,4})\s*mm", t["text"], re.IGNORECASE)
+ cm = re.findall(r"(\d{2,4})\s*cm", t["text"], re.IGNORECASE)
if cm:
html_width = f"{cm[0]}cm"
elif mm:
@@ -105,7 +115,7 @@ def parse_page(url, h):
if "repeat" in title_l or (t["tag"] == "p" and "repeat" in (t.get("text") or "").lower()):
html_repeat = html_repeat or t["text"]
mt = re.search(r"(straight match|half drop|drop match|free match|random match|offset match|no match|reverse hang)",
- t["text"], re.I)
+ t["text"], re.IGNORECASE)
if mt:
html_match = mt.group(1).title()
@@ -132,16 +142,16 @@ def pdf_to_blob(pdf_path):
return re.sub(r"\s+", "", raw)
FIRE_CODES = [
- (re.compile(r"euroclass\s*([a-f])", re.I), lambda m: f"Euroclass {m.group(1).upper()}"),
- (re.compile(r"astm\s*e-?84\D{0,10}class\s*([a-c])", re.I), lambda m: f"ASTM E84 Class {m.group(1).upper()}"),
- (re.compile(r"class\s*([a-c])\D{0,20}astm\s*e-?84", re.I), lambda m: f"ASTM E84 Class {m.group(1).upper()}"),
- (re.compile(r"astm\s*e-?84", re.I), lambda m: "ASTM E84"),
- (re.compile(r"bs\s*476[\s\-]*part\s*(\d+)", re.I), lambda m: f"BS 476 Part {m.group(1)}"),
- (re.compile(r"bs\s*en\s*476", re.I), lambda m: "BS EN 476"),
- (re.compile(r"bs\s*476", re.I), lambda m: "BS 476"),
- (re.compile(r"en\s*13501[\s\-]*1", re.I), lambda m: "EN 13501-1"),
- (re.compile(r"nfpa\s*701", re.I), lambda m: "NFPA 701"),
- (re.compile(r"\bimo\b", re.I), lambda m: "IMO"),
+ (re.compile(r"euroclass\s*([a-f])", re.IGNORECASE), lambda m: f"Euroclass {m.group(1).upper()}"),
+ (re.compile(r"astm\s*e-?84\D{0,10}class\s*([a-c])", re.IGNORECASE), lambda m: f"ASTM E84 Class {m.group(1).upper()}"),
+ (re.compile(r"class\s*([a-c])\D{0,20}astm\s*e-?84", re.IGNORECASE), lambda m: f"ASTM E84 Class {m.group(1).upper()}"),
+ (re.compile(r"astm\s*e-?84", re.IGNORECASE), lambda m: "ASTM E84"),
+ (re.compile(r"bs\s*476[\s\-]*part\s*(\d+)", re.IGNORECASE), lambda m: f"BS 476 Part {m.group(1)}"),
+ (re.compile(r"bs\s*en\s*476", re.IGNORECASE), lambda m: "BS EN 476"),
+ (re.compile(r"bs\s*476", re.IGNORECASE), lambda m: "BS 476"),
+ (re.compile(r"en\s*13501[\s\-]*1", re.IGNORECASE), lambda m: "EN 13501-1"),
+ (re.compile(r"nfpa\s*701", re.IGNORECASE), lambda m: "NFPA 701"),
+ (re.compile(r"\bimo\b", re.IGNORECASE), lambda m: "IMO"),
]
MATCH_TERMS = ["Straight Match", "Half Drop", "Drop Match", "Free Match", "Random Match",
@@ -149,7 +159,7 @@ MATCH_TERMS = ["Straight Match", "Half Drop", "Drop Match", "Free Match", "Rando
def extract_window(blob, label_words, window=260):
for label in label_words:
- pat = re.compile(fuzzy_ti(label), re.I)
+ pat = re.compile(fuzzy_ti(label), re.IGNORECASE)
m = pat.search(blob)
if m:
return blob[m.end(): m.end() + window]
@@ -185,14 +195,14 @@ def extract_finish(blob):
return ""
def extract_application_from_filename(pdf_url):
- m = re.search(r"type[-\s]?(i{1,3})\b", pdf_url, re.I)
+ m = re.search(r"type[-\s]?(i{1,3})\b", pdf_url, re.IGNORECASE)
if m:
n = len(m.group(1))
return f"Type {'I'*n}"
return ""
def extract_application_from_blob(blob):
- m = re.search(r"type(iii|ii|i)\b", blob, re.I)
+ m = re.search(r"type(iii|ii|i)\b", blob, re.IGNORECASE)
if m:
roman = m.group(1).upper()
return f"Type {roman}"
@@ -208,15 +218,15 @@ def extract_match(blob):
return "; ".join(found)
def extract_repeat(blob):
- m = re.search(r"\(?(\d+(?:\.\d+)?)\s*(mm|cm|in)\s*repeat", blob, re.I)
+ m = re.search(r"\(?(\d+(?:\.\d+)?)\s*(mm|cm|in)\s*repeat", blob, re.IGNORECASE)
if not m:
- m = re.search(r"repeat\D{0,10}(\d+(?:\.\d+)?)\s*(mm|cm|in)", blob, re.I)
+ m = re.search(r"repeat\D{0,10}(\d+(?:\.\d+)?)\s*(mm|cm|in)", blob, re.IGNORECASE)
if m:
return f"{m.group(1)}{m.group(2).lower()}"
return ""
def extract_roll_size(blob):
- m = re.search(r"(\d{2,4})\s*cm\s*x\s*(\d{1,3}(?:\.\d+)?)\s*m\b", blob, re.I)
+ m = re.search(r"(\d{2,4})\s*cm\s*x\s*(\d{1,3}(?:\.\d+)?)\s*m\b", blob, re.IGNORECASE)
if m:
return m.group(1), m.group(2)
return None, None
@@ -320,7 +330,7 @@ def crawl():
if not merged["match_type"] and pg["html_match"]:
merged["match_type"] = pg["html_match"]
if not merged["repeat"] and pg["html_repeat"]:
- rm = re.search(r"(\d+(?:\.\d+)?)\s*(mm|cm|in)", pg["html_repeat"], re.I)
+ rm = re.search(r"(\d+(?:\.\d+)?)\s*(mm|cm|in)", pg["html_repeat"], re.IGNORECASE)
if rm:
merged["repeat"] = f"{rm.group(1)}{rm.group(2).lower()}"
results.append({
← 4b0aa1e TK-11424: backfill Newmor spec metafields to live Shopify (S
·
back to Newmor Onboard
·
auto-data-snapshot: 2026-09-14T16:46:18 (1 data files) — scr 9853f75 →