← back to Dw Material Envelope Canary
canary: robust envelope detector + weekly all-metafields bulk pass (contrarian hardening)
f11d1f0a2de90fec63b15816dc8542d68a5b44a4 · 2026-07-07 10:37:59 -0700 · steve
Files touched
A __pycache__/check-bulk.cpython-314.pycA __pycache__/check.cpython-314.pycA check-bulk.pyM check.py
Diff
commit f11d1f0a2de90fec63b15816dc8542d68a5b44a4
Author: steve <steve@designerwallcoverings.com>
Date: Tue Jul 7 10:37:59 2026 -0700
canary: robust envelope detector + weekly all-metafields bulk pass (contrarian hardening)
---
__pycache__/check-bulk.cpython-314.pyc | Bin 0 -> 7381 bytes
__pycache__/check.cpython-314.pyc | Bin 0 -> 8613 bytes
check-bulk.py | 65 +++++++++++++++++++++++++++++++++
check.py | 9 +++--
4 files changed, 71 insertions(+), 3 deletions(-)
diff --git a/__pycache__/check-bulk.cpython-314.pyc b/__pycache__/check-bulk.cpython-314.pyc
new file mode 100644
index 0000000..b2989af
Binary files /dev/null and b/__pycache__/check-bulk.cpython-314.pyc differ
diff --git a/__pycache__/check.cpython-314.pyc b/__pycache__/check.cpython-314.pyc
new file mode 100644
index 0000000..49cd414
Binary files /dev/null and b/__pycache__/check.cpython-314.pyc differ
diff --git a/check-bulk.py b/check-bulk.py
new file mode 100644
index 0000000..e6cbb33
--- /dev/null
+++ b/check-bulk.py
@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""WEEKLY exhaustive pass — the durable net the daily check.py can't be.
+Runs a Shopify bulk operation exporting EVERY metafield on EVERY product, then
+scans every value with the robust substring detector (catches scalar, list, and
+double-escaped envelope variants). No curated key list. CNCP alert on FAIL.
+READ-ONLY. Born 2026-07-07 from the /yolo contrarian's 'watch all keys' verdict."""
+import os, json, sys, time, urllib.request, datetime
+HERE = os.path.dirname(os.path.abspath(__file__))
+def load_env():
+ for p in [os.path.expanduser("~/Projects/Designer-Wallcoverings/.env"), os.path.join(HERE, ".env")]:
+ if os.path.exists(p):
+ for ln in open(p):
+ if "=" in ln and not ln.strip().startswith("#"):
+ k, v = ln.strip().split("=", 1); os.environ.setdefault(k, v.strip().strip('"').strip("'"))
+load_env()
+DOMAIN = os.environ.get("SHOPIFY_STORE_DOMAIN", "designer-laboratory-sandbox.myshopify.com")
+TOK = os.environ.get("SHOPIFY_ADMIN_TOKEN") or os.environ.get("SHOPIFY_ADMIN_ACCESS_TOKEN")
+URL = f"https://{DOMAIN}/admin/api/2024-01/graphql.json"
+CNCP = "http://127.0.0.1:3333/api/parking_lot"
+def gql(q):
+ b = json.dumps({"query": q}).encode()
+ return json.loads(urllib.request.urlopen(urllib.request.Request(
+ URL, data=b, headers={"X-Shopify-Access-Token": TOK, "Content-Type": "application/json"}), timeout=60).read())
+def is_env(v):
+ return isinstance(v, str) and "single_line_text_field" in v
+START = 'mutation { bulkOperationRunQuery(query: """{ products { edges { node { id handle metafields { edges { node { namespace key value } } } } } } }""") { bulkOperation { id status } userErrors { message } } }'
+POLL = '{ currentBulkOperation { status url objectCount } }'
+def main():
+ r = gql(START)
+ ue = r["data"]["bulkOperationRunQuery"]["userErrors"]
+ if ue: print("start err", ue); return 2
+ while True:
+ op = gql(POLL)["data"]["currentBulkOperation"]
+ if op["status"] in ("COMPLETED", "FAILED", "CANCELED"): break
+ time.sleep(20)
+ if op["status"] != "COMPLETED" or not op.get("url"):
+ print("bulk not completed:", op); return 2
+ path = os.path.join(HERE, "data", "bulk.jsonl")
+ urllib.request.urlretrieve(op["url"], path)
+ hits = {}; nprod = 0; nmf = 0
+ for ln in open(path):
+ try: o = json.loads(ln)
+ except: continue
+ if "namespace" in o and "key" in o:
+ nmf += 1
+ if is_env(o.get("value")):
+ hits.setdefault(f'{o["namespace"]}.{o["key"]}', 0); hits[f'{o["namespace"]}.{o["key"]}'] += 1
+ elif "handle" in o: nprod += 1
+ os.remove(path) # 800MB — don't keep
+ verdict = "FAIL" if hits else "PASS"
+ json.dump({"ts": datetime.datetime.now().isoformat(), "products": nprod, "metafields": nmf,
+ "verdict": verdict, "hits": hits}, open(os.path.join(HERE, "data", "latest-bulk.json"), "w"), indent=1)
+ print(f"{verdict}: {nprod} products, {nmf} metafields, {sum(hits.values())} enveloped across {len(hits)} keys")
+ if hits:
+ try:
+ card = {"title": f"⚠️ Metafield envelope regression (weekly bulk): {sum(hits.values())} values across {len(hits)} keys",
+ "note": f"dw-material-envelope-canary weekly bulk scan found leaked JSON envelopes: {json.dumps(hits)[:300]}. Re-run the unwrap fix + find the importer.",
+ "source": "dw-material-envelope-canary-bulk"}
+ urllib.request.urlopen(urllib.request.Request(CNCP, data=json.dumps(card).encode(),
+ headers={"Content-Type": "application/json"}), timeout=5)
+ print(" posted CNCP alert")
+ except Exception as ex: print(" CNCP post failed:", ex)
+ return 1 if hits else 0
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/check.py b/check.py
index dd74184..5872ac9 100644
--- a/check.py
+++ b/check.py
@@ -26,7 +26,8 @@ TOK = os.environ.get("SHOPIFY_ADMIN_TOKEN") or os.environ.get("SHOPIFY_ADMIN_ACC
URL = f"https://{DOMAIN}/admin/api/2024-01/graphql.json"
CNCP = "http://127.0.0.1:3333/api/parking_lot"
-# The known-corruptible keys. Add more here if the bug ever appears elsewhere.
+# Daily FAST pass: the 3 keys the bug actually hit (cheap GraphQL, quick signal).
+# The WEEKLY check-bulk.py covers ALL metafields exhaustively (the durable net).
CHECK_KEYS = [("custom", "material"), ("dwc", "contents"), ("global", "Material")]
FIELDS = " ".join(f'k{i}: metafield(namespace:"{ns}", key:"{k}"){{ value }}'
for i, (ns, k) in enumerate(CHECK_KEYS))
@@ -48,8 +49,10 @@ def gql(cursor):
raise SystemExit("gql failed")
def is_env(v):
- v = (v or "").strip()
- return v.startswith("{") and "single_line_text_field" in v
+ # robust: the metafield TYPE NAME never legitimately appears in a spec value,
+ # so any value containing it is a leaked envelope — catches scalar, list,
+ # and double-escaped (\"single_line_text_field\") variants alike.
+ return isinstance(v, str) and "single_line_text_field" in v
def main():
total = 0; bad = []
← 6e5b25e dw-material-envelope-canary: recurrence monitor for metafiel
·
back to Dw Material Envelope Canary
·
chore: lint, refactor, TOK guard, v1.0.1 (session close) 1a28aa5 →