← back to Gmc Titlefix
TK-11233: fix two fail-closed selector bugs; add authoritative LPE classifier
f8d022d8fa9ecc675c718fad73c8ee8978074b24 · 2026-09-10 08:04:36 -0700 · Steve Abrams
Selector bugs, both of which produced a FALSE '0 candidates':
* psql -F tab separator was shell-mangled to a literal backslash-t, so no row
split and all 1,916 classed NO-PRODUCT-MFR-SKU. Now execFileSync with an
args array (no shell) plus a 7-field row assertion that throws rather than
proceeding on a malformed identity source.
* shopify_products.shopify_id is mixed-format (bare numeric OR full GID), so
the GID was double-prefixed and every Shopify read errored. toProductGid
normalises either form and throws if no numeric id can be derived.
Both failed closed, never toward a bad write - but the zero was a bug, not an
answer, so skip reasons must always be inspected. Post-fix the selector finds
real work: 400px primaries against 1800-4480px originals.
LPE classifier reads the root cause from the Shopify Admin API (an ACTIVE
product not published to the Online Store has no PDP, so /products/<handle>
302s to home) instead of probing the live storefront, which rate-limits and
serves different responses to different clients.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W43bYwipr6GHKjJtWSsMyX
Files touched
A tk11233-lpe-classify.mjsM tk11233-umbrella-hires-v2.mjs
Diff
commit f8d022d8fa9ecc675c718fad73c8ee8978074b24
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 10 08:04:36 2026 -0700
TK-11233: fix two fail-closed selector bugs; add authoritative LPE classifier
Selector bugs, both of which produced a FALSE '0 candidates':
* psql -F tab separator was shell-mangled to a literal backslash-t, so no row
split and all 1,916 classed NO-PRODUCT-MFR-SKU. Now execFileSync with an
args array (no shell) plus a 7-field row assertion that throws rather than
proceeding on a malformed identity source.
* shopify_products.shopify_id is mixed-format (bare numeric OR full GID), so
the GID was double-prefixed and every Shopify read errored. toProductGid
normalises either form and throws if no numeric id can be derived.
Both failed closed, never toward a bad write - but the zero was a bug, not an
answer, so skip reasons must always be inspected. Post-fix the selector finds
real work: 400px primaries against 1800-4480px originals.
LPE classifier reads the root cause from the Shopify Admin API (an ACTIVE
product not published to the Online Store has no PDP, so /products/<handle>
302s to home) instead of probing the live storefront, which rate-limits and
serves different responses to different clients.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01W43bYwipr6GHKjJtWSsMyX
---
tk11233-lpe-classify.mjs | 111 ++++++++++++++++++++++++++++++++++++++++++
tk11233-umbrella-hires-v2.mjs | 11 ++++-
2 files changed, 121 insertions(+), 1 deletion(-)
diff --git a/tk11233-lpe-classify.mjs b/tk11233-lpe-classify.mjs
new file mode 100644
index 0000000..97644aa
--- /dev/null
+++ b/tk11233-lpe-classify.mjs
@@ -0,0 +1,111 @@
+#!/usr/bin/env node
+/**
+ * TK-11233 - READ-ONLY authoritative classification of the landing_page_error cohort.
+ *
+ * Supersedes storefront probing. Probing the live PDP is both slower and worse evidence: the
+ * storefront rate-limits (429) and serves different responses to different clients, and it puts
+ * load on Steve's customer-facing site. The ROOT CAUSE is a Shopify channel fact, so read it from
+ * the Admin API instead:
+ *
+ * an ACTIVE product that is NOT published to the Online Store publication has no PDP, so
+ * /products/<handle> 302s to the homepage -> Google records landing_page_error.
+ *
+ * Verified 2026-09-10: dwqw-58842-handle = ACTIVE, publishedOnPublication(Online Store)=false,
+ * and node fetch confirms 302 -> https://www.designerwallcoverings.com/.
+ *
+ * Each cohort item is classified into the remediation lane it belongs to:
+ * OFF-ONLINE-STORE-in-google product is off Online Store but still on Google & YouTube
+ * -> the live defect. Two sub-lanes:
+ * * peel-and-stick -> UNPUBLISH from Google (line lives on
+ * apartmentwallpaper.com, per B1)
+ * * regular -> PUBLISH to Online Store (per B1)
+ * PUBLISHED-BOTH product is on Online Store and on Google -> the PDP resolves;
+ * the disapproval is stale and clears on recrawl. No action.
+ * NOT-ACTIVE / MISSING product archived/draft/deleted but still in the feed -> feed
+ * hygiene, handled by the existing orphan reconcile lane.
+ *
+ * Fires NOTHING. $0. Admin API reads only; the storefront is never touched.
+ */
+import { createRequire } from 'module';
+const require = createRequire(import.meta.url);
+import fs from 'fs';
+
+const SHOP = 'designer-laboratory-sandbox';
+const IN = '/Users/macstudio3/Projects/gmc-titlefix/data/tk11233-lpe-cohort.json';
+const OUT = '/Users/macstudio3/Projects/gmc-titlefix/data/tk11233-lpe-classified.json';
+const PUB_ONLINE_STORE = 'gid://shopify/Publication/22208643184';
+const PUB_GOOGLE = 'gid://shopify/Publication/29646651457';
+
+const tok = (fs.readFileSync(process.env.HOME + '/Projects/secrets-manager/.env', 'utf8').match(/^SHOPIFY_FULL_ACCESS_TOKEN=(.+)$/m) || [])[1];
+if (!tok) { console.error('missing SHOPIFY_FULL_ACCESS_TOKEN'); process.exit(2); }
+
+async function gql(query, variables) {
+ for (let a = 0; a < 5; a++) {
+ const r = await fetch('https://' + SHOP + '.myshopify.com/admin/api/2024-10/graphql.json', {
+ method: 'POST', headers: { 'X-Shopify-Access-Token': tok, 'Content-Type': 'application/json' },
+ body: JSON.stringify({ query, variables }), signal: AbortSignal.timeout(60000)
+ });
+ const j = await r.json();
+ if (j.errors) {
+ const throttled = JSON.stringify(j.errors).includes('THROTTLED');
+ if (throttled && a < 4) { await new Promise(z => setTimeout(z, 2000 * (a + 1))); continue; }
+ throw new Error(JSON.stringify(j.errors).slice(0, 300));
+ }
+ return j.data;
+ }
+ throw new Error('throttled out');
+}
+
+const handleOf = link => { try { return (new URL(link).pathname.match(/\/products\/([^/]+)/) || [])[1] || null; } catch { return null; } };
+const isPeelStick = (title, handle) => /peel[\s-]?and[\s-]?stick|removable|surface stick|\bsur-\d+/i.test((title || '') + ' ' + (handle || ''));
+
+(async () => {
+ const cohort = JSON.parse(fs.readFileSync(IN, 'utf8')).items;
+ const rows = [];
+ let done = 0;
+ const CONC = 4;
+ let idx = 0;
+ async function worker() {
+ while (idx < cohort.length) {
+ const it = cohort[idx++];
+ const h = handleOf(it.link);
+ const row = { offerId: it.offerId, link: it.link, handle: h, feed_title: it.title };
+ if (!h) { row.cls = 'NO-HANDLE-IN-LINK'; rows.push(row); continue; }
+ try {
+ const d = await gql('query($h:String!,$a:ID!,$b:ID!){ productByHandle(handle:$h){ id title status vendor onlineStore:publishedOnPublication(publicationId:$a) google:publishedOnPublication(publicationId:$b) } }',
+ { h, a: PUB_ONLINE_STORE, b: PUB_GOOGLE });
+ const p = d.productByHandle;
+ if (!p) { row.cls = 'MISSING-PRODUCT'; }
+ else {
+ Object.assign(row, { product_id: p.id, title: p.title, vendor: p.vendor, status: p.status, on_online_store: p.onlineStore, on_google: p.google });
+ if (p.status !== 'ACTIVE') row.cls = 'NOT-ACTIVE-' + p.status;
+ else if (!p.onlineStore) {
+ row.cls = 'OFF-ONLINE-STORE-in-google';
+ row.lane = isPeelStick(p.title, h) ? 'peel-and-stick -> UNPUBLISH from Google' : 'regular -> PUBLISH to Online Store';
+ } else row.cls = p.google ? 'PUBLISHED-BOTH-stale-disapproval' : 'PUBLISHED-ONLINE-STORE-not-on-google';
+ }
+ } catch (e) { row.cls = 'ERROR'; row.err = String(e).slice(0, 140); }
+ rows.push(row);
+ if (++done % 250 === 0) process.stderr.write(' ' + done + '/' + cohort.length + '\n');
+ await new Promise(z => setTimeout(z, 120));
+ }
+ }
+ await Promise.all(Array.from({ length: CONC }, worker));
+
+ const hist = {}, lanes = {}, vendors = {};
+ for (const r of rows) {
+ hist[r.cls] = (hist[r.cls] || 0) + 1;
+ if (r.lane) lanes[r.lane] = (lanes[r.lane] || 0) + 1;
+ if (r.cls === 'OFF-ONLINE-STORE-in-google') vendors[r.vendor || '(none)'] = (vendors[r.vendor || '(none)'] || 0) + 1;
+ }
+ const doc = {
+ ticket: 'TK-11233', bucket: 'landing_page_error', method: 'shopify-admin-publication-read (no storefront load)',
+ classified_at: new Date().toISOString(), total: rows.length,
+ histogram: hist, remediation_lanes: lanes,
+ off_channel_by_vendor: Object.entries(vendors).sort((a, b) => b[1] - a[1]).slice(0, 25),
+ rows
+ };
+ fs.writeFileSync(OUT, JSON.stringify(doc, null, 2));
+ console.log(JSON.stringify({ total: rows.length, histogram: hist, remediation_lanes: lanes, off_channel_by_vendor: doc.off_channel_by_vendor.slice(0, 12) }, null, 2));
+ console.log('-> ' + OUT);
+})();
diff --git a/tk11233-umbrella-hires-v2.mjs b/tk11233-umbrella-hires-v2.mjs
index 66c1bfa..e49130c 100644
--- a/tk11233-umbrella-hires-v2.mjs
+++ b/tk11233-umbrella-hires-v2.mjs
@@ -130,8 +130,17 @@ function dims(url) {
} catch { return { w: 0, h: 0, long: 0 }; }
finally { try { fs.unlinkSync(tmp); } catch { /* already gone */ } }
}
+// shopify_products.shopify_id is MIXED FORMAT in dw_unified: some rows hold a bare numeric id,
+// others a full 'gid://shopify/Product/<n>'. Normalise to a GID either way.
+function toProductGid(shopifyId) {
+ const s = String(shopifyId || '').trim();
+ const n = (s.match(/(\d+)\s*$/) || [])[1];
+ if (!n) throw new Error('cannot derive a numeric product id from shopify_id ' + JSON.stringify(s));
+ return 'gid://shopify/Product/' + n;
+}
+
async function productMedia(shopifyId) {
- const gid = 'gid://shopify/Product/' + shopifyId;
+ const gid = toProductGid(shopifyId);
const d = await shopify('query($id:ID!){ product(id:$id){ id title status media(first:50){ nodes{ id mediaContentType ... on MediaImage { status image { url width height } } } } } }', { id: gid });
const p = d.product; if (!p) return null;
const imgs = p.media.nodes.filter(n => n.mediaContentType === 'IMAGE');
← 57dde34 TK-11233: identity-validated, shortlist-bound umbrella hi-re
·
back to Gmc Titlefix
·
TK-11233: authoritative landing_page_error classification of 7877b0d →