← back to Letsbegin
rw-price-group-slice17.js
467 lines
#!/usr/bin/env node
/**
* rw-price-group-slice17.js — Price-group detector + cost-metafield writer
* for SLICE 17 (OFFSET 640, LIMIT 40) of the Rebel Walls mural cohort.
*
* Algorithm:
* 1. For each shopify_id, fetch custom.rw_product_url + custom.price_group.
* 2. If no URL -> count unknown, skip.
* 3. curl the RW page, extract base per-m2 price from JSON-LD or price chip.
* 4. Bucket: ~58.10 -> C2, ~76.40 -> C3, ~88.30 -> C4 (nearest).
* 5. Write price_group + (if != C3) cost_per_m2 + material_options + cost_basis.
* If C3, write price_group=C3 but leave cost_per_m2/material_options unchanged.
* 6. Report {processed, c2, c3, c4, unknown, failed}.
*
* SANDBOX ONLY. Rate-limited ~2 req/s.
*/
const https = require('https');
const http = require('http');
const { URL } = require('url');
// ---- env ----
const fs = require('fs');
function loadEnv(path) {
try {
for (const line of fs.readFileSync(path, 'utf8').split('\n')) {
const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)$/);
if (!m) continue;
let v = m[2].trim();
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
if (process.env[m[1]] == null) process.env[m[1]] = v;
}
} catch { /* ok */ }
}
loadEnv('/Users/macstudio3/Projects/secrets-manager/.env');
const STORE = process.env.SHOPIFY_STORE || 'designer-laboratory-sandbox.myshopify.com';
const TOKEN = process.env.SHOPIFY_ADMIN_TOKEN || process.env.SHOPIFY_PRODUCT_TOKEN;
const API = '/admin/api/2024-10/graphql.json';
if (!TOKEN) { console.error('FATAL: SHOPIFY_ADMIN_TOKEN required'); process.exit(1); }
if (STORE !== 'designer-laboratory-sandbox.myshopify.com') {
console.error(`FATAL: refusing to run against non-sandbox store "${STORE}"`); process.exit(1);
}
// ---- the 40 IDs for slice 17 (OFFSET 640, LIMIT 40) ----
const SLICE_IDS = [
'gid://shopify/Product/7855826075699',
'gid://shopify/Product/7855826141235',
'gid://shopify/Product/7855826206771',
'gid://shopify/Product/7855826239539',
'gid://shopify/Product/7855826272307',
'gid://shopify/Product/7855826370611',
'gid://shopify/Product/7855826436147',
'gid://shopify/Product/7855826468915',
'gid://shopify/Product/7855826534451',
'gid://shopify/Product/7855826567219',
'gid://shopify/Product/7855826599987',
'gid://shopify/Product/7855826665523',
'gid://shopify/Product/7855826698291',
'gid://shopify/Product/7855826731059',
'gid://shopify/Product/7855826829363',
'gid://shopify/Product/7855826862131',
'gid://shopify/Product/7855826927667',
'gid://shopify/Product/7855826960435',
'gid://shopify/Product/7855826993203',
'gid://shopify/Product/7855827025971',
'gid://shopify/Product/7855827058739',
'gid://shopify/Product/7855827124275',
'gid://shopify/Product/7855827157043',
'gid://shopify/Product/7855827222579',
'gid://shopify/Product/7855827255347',
'gid://shopify/Product/7855827288115',
'gid://shopify/Product/7855827353651',
'gid://shopify/Product/7855827386419',
'gid://shopify/Product/7855827419187',
'gid://shopify/Product/7855827451955',
'gid://shopify/Product/7855827517491',
'gid://shopify/Product/7855827550259',
'gid://shopify/Product/7855827583027',
'gid://shopify/Product/7855827615795',
'gid://shopify/Product/7855827681331',
'gid://shopify/Product/7855827714099',
'gid://shopify/Product/7855827779635',
'gid://shopify/Product/7855827812403',
'gid://shopify/Product/7855827845171',
'gid://shopify/Product/7855827877939',
];
// ---- price group data ----
const GROUPS = {
C2: {
cost_per_m2: '33.84',
material_options: JSON.stringify([
{"material":"Non-Woven (Standard)","retail_per_m2":58.10,"cost_per_m2":33.84,"cost_confirmed":false},
{"material":"Peel & Stick","retail_per_m2":69.70,"cost_per_m2":40.60,"cost_confirmed":false},
{"material":"Commercial Grade","retail_per_m2":81.36,"cost_per_m2":47.39,"cost_confirmed":false}
]),
retail: 58.10,
cost_basis: 'C2 price group; Non-Woven RRP $58.10/m2; cost = RRP x 0.5825; P&S/Commercial extrapolated. Sourced from live RW public product page.',
},
C3: {
cost_per_m2: '44.50',
material_options: JSON.stringify([
{"material":"Non-Woven (Standard)","retail_per_m2":76.40,"cost_per_m2":44.50,"cost_confirmed":false},
{"material":"Peel & Stick","retail_per_m2":91.70,"cost_per_m2":53.42,"cost_confirmed":false},
{"material":"Commercial Grade","retail_per_m2":107.00,"cost_per_m2":62.33,"cost_confirmed":false}
]),
retail: 76.40,
cost_basis: 'C3 price group; Non-Woven RRP $76.40/m2; cost = RRP x 0.5825; P&S/Commercial extrapolated. Sourced from live RW public product page.',
},
C4: {
cost_per_m2: '51.44',
material_options: JSON.stringify([
{"material":"Non-Woven (Standard)","retail_per_m2":88.30,"cost_per_m2":51.44,"cost_confirmed":false},
{"material":"Peel & Stick","retail_per_m2":106.00,"cost_per_m2":61.75,"cost_confirmed":false},
{"material":"Commercial Grade","retail_per_m2":123.66,"cost_per_m2":72.03,"cost_confirmed":false}
]),
retail: 88.30,
cost_basis: 'C4 price group; Non-Woven RRP $88.30/m2; cost = RRP x 0.5825; P&S/Commercial extrapolated. Sourced from live RW public product page.',
},
};
function bucketPrice(pricePerM2) {
// Snap to nearest anchor: C2=58.10, C3=76.40, C4=88.30
const anchors = [
{ group: 'C2', anchor: 58.10 },
{ group: 'C3', anchor: 76.40 },
{ group: 'C4', anchor: 88.30 },
];
let best = anchors[0];
let bestDist = Math.abs(pricePerM2 - anchors[0].anchor);
for (const a of anchors) {
const d = Math.abs(pricePerM2 - a.anchor);
if (d < bestDist) { bestDist = d; best = a; }
}
return best.group;
}
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
// ---- Shopify GraphQL ----
function gqlRaw(body) {
return new Promise((resolve, reject) => {
const data = JSON.stringify(body);
const req = https.request({
hostname: STORE, path: API, method: 'POST',
headers: { 'X-Shopify-Access-Token': TOKEN, 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data) },
}, res => { let c = ''; res.on('data', d => c += d); res.on('end', () => { try { resolve(JSON.parse(c)); } catch { resolve({ error: c.slice(0, 300) }); } }); });
req.on('error', reject);
req.setTimeout(60000, () => { req.destroy(); reject(new Error('timeout')); });
req.write(data); req.end();
});
}
async function gql(body, retries = 5) {
for (let attempt = 1; attempt <= retries; attempt++) {
try {
const result = await gqlRaw(body);
const throttled = result?.errors?.some(e => /Throttled/i.test(e.message || ''));
const lowBudget = (result?.extensions?.cost?.throttleStatus?.currentlyAvailable || 9999) < 200;
if (throttled) { await sleep(3000); continue; }
if (lowBudget) await sleep(1500);
return result;
} catch (e) {
if (attempt < retries) { await sleep(attempt * 2000); continue; }
throw e;
}
}
}
const MF_MUTATION = 'mutation metafieldsSet($m: [MetafieldsSetInput!]!) { metafieldsSet(metafields: $m) { metafields { key } userErrors { message field } } }';
async function fetchMetafields(pid) {
const r = await gql({ query: `{ product(id:"${pid}") {
id title vendor
url: metafield(namespace:"custom", key:"rw_product_url"){ value }
pg: metafield(namespace:"custom", key:"price_group"){ value }
cost: metafield(namespace:"custom", key:"cost_per_m2"){ value }
} }` });
const p = r?.data?.product;
if (!p) return null;
return {
id: p.id,
title: p.title,
vendor: p.vendor,
rw_product_url: p.url?.value ?? null,
price_group: p.pg?.value ?? null,
cost_per_m2: p.cost?.value ?? null,
};
}
// ---- fetch RW product page and extract base price ----
function fetchUrl(urlStr, redirects = 5) {
return new Promise((resolve, reject) => {
try {
const u = new URL(urlStr);
const lib = u.protocol === 'https:' ? https : http;
const req = lib.request({
hostname: u.hostname,
path: u.pathname + u.search,
method: 'GET',
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/124 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,*/*;q=0.9',
'Accept-Language': 'en-US,en;q=0.9',
},
}, res => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && redirects > 0) {
res.resume(); // drain
resolve(fetchUrl(res.headers.location, redirects - 1));
return;
}
let c = '';
let truncated = false;
res.setEncoding('utf8');
res.on('data', d => {
if (!truncated) {
c += d;
if (c.length > 600000) {
truncated = true;
res.destroy(); // destroys the socket, triggers 'close', not 'end'
}
}
});
res.on('end', () => resolve({ status: res.statusCode, body: c }));
res.on('close', () => { if (truncated) resolve({ status: res.statusCode, body: c }); });
res.on('error', () => { if (truncated) resolve({ status: res.statusCode || 200, body: c }); });
});
req.on('error', (e) => { reject(e); });
req.setTimeout(45000, () => { req.destroy(); reject(new Error('timeout fetching ' + urlStr)); });
req.end();
} catch (e) { reject(e); }
});
}
function extractPriceFromPage(html, urlStr) {
// Strategy 1: JSON-LD with offers.price
const jsonldMatches = [...html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi)];
for (const m of jsonldMatches) {
try {
const obj = JSON.parse(m[1]);
const items = Array.isArray(obj) ? obj : [obj];
for (const item of items) {
const price = item?.offers?.price ?? item?.offers?.[0]?.price;
if (price != null && !isNaN(parseFloat(price))) {
const p = parseFloat(price);
if (p >= 2 && p <= 20) return p * 10.7639; // sqft -> m2
if (p >= 40 && p <= 200) return p; // already m2
}
}
} catch { /* ignore */ }
}
// Strategy 2: "From $X / sq ft" patterns
const sqftPatterns = [
/From\s+\$([0-9]+\.?[0-9]*)\s*\/\s*sq\.?\s*ft/i,
/\$([0-9]+\.?[0-9]*)\s*\/\s*sq\.?\s*ft/i,
/From\s+USD\s*([0-9]+\.?[0-9]*)\s*\/\s*sq/i,
];
for (const pat of sqftPatterns) {
const m = html.match(pat);
if (m) {
const p = parseFloat(m[1]);
if (p >= 2 && p <= 20) return p * 10.7639;
}
}
// Strategy 3: per m2 patterns
const m2Patterns = [
/From\s+\$([0-9]+\.?[0-9]*)\s*\/\s*m[²2]/i,
/\$([0-9]+\.?[0-9]*)\s*\/\s*m[²2]/i,
/From\s+USD\s*([0-9]+\.?[0-9]*)\s*\/\s*m[²2]/i,
];
for (const pat of m2Patterns) {
const m = html.match(pat);
if (m) {
const p = parseFloat(m[1]);
if (p >= 40 && p <= 200) return p;
}
}
// Strategy 4: data attributes or JS variables with price
const priceAttrPatterns = [
/"price"\s*:\s*([0-9]+\.?[0-9]*)/,
/data-price="([0-9]+\.?[0-9]*)"/,
/'price'\s*:\s*([0-9]+\.?[0-9]*)/,
/price_per_sqft['":\s]+([0-9]+\.?[0-9]*)/i,
/pricePerSqFt['":\s]+([0-9]+\.?[0-9]*)/i,
/basePrice["':\s]+([0-9]+\.?[0-9]*)/i,
];
for (const pat of priceAttrPatterns) {
const m = html.match(pat);
if (m) {
const p = parseFloat(m[1]);
if (p >= 2 && p <= 20) return p * 10.7639;
if (p >= 40 && p <= 200) return p;
}
}
// Strategy 5: look for standalone price-like numbers near pricing keywords
const priceSectionMatch = html.match(/(?:From|Starting|Price|Cost)[^$]*\$([0-9]+\.[0-9]+)/i);
if (priceSectionMatch) {
const p = parseFloat(priceSectionMatch[1]);
if (p >= 2 && p <= 20) return p * 10.7639;
if (p >= 40 && p <= 200) return p;
}
// Strategy 6: look for the specific RW anchor prices directly
// Check for known price patterns: 5.40, 7.10, 8.20, 58.10, 76.40, 88.30
const knownPrices = [
{ sqft: 5.40, group: 'C2' },
{ sqft: 7.10, group: 'C3' },
{ sqft: 8.20, group: 'C4' },
];
for (const kp of knownPrices) {
const re = new RegExp('\\$' + kp.sqft.toFixed(2).replace('.', '\\.'));
if (re.test(html)) return kp.sqft * 10.7639;
}
return null;
}
async function writeMetafields(pid, group) {
const g = GROUPS[group];
const mf = [
{ ownerId: pid, namespace: 'custom', key: 'price_group', value: group, type: 'single_line_text_field' },
{ ownerId: pid, namespace: 'custom', key: 'cost_basis', value: g.cost_basis, type: 'multi_line_text_field' },
];
// For C2 and C4, overwrite cost_per_m2 and material_options.
// For C3, leave cost/material_options as-is (they're already the C3 defaults).
if (group !== 'C3') {
mf.push({ ownerId: pid, namespace: 'custom', key: 'cost_per_m2', value: g.cost_per_m2, type: 'number_decimal' });
mf.push({ ownerId: pid, namespace: 'custom', key: 'material_options', value: g.material_options, type: 'json' });
}
const r = await gql({ query: MF_MUTATION, variables: { m: mf } });
const errs = r?.data?.metafieldsSet?.userErrors || [];
return errs.map(e => `${(e.field || []).join('.')}: ${e.message}`);
}
async function main() {
const counts = { processed: 0, c2: 0, c3: 0, c4: 0, unknown: 0, failed: 0 };
const failureLog = [];
const total = SLICE_IDS.length;
console.log(`[rw-price-group-slice17] Processing ${total} products (OFFSET 640, LIMIT 40)`);
console.log(`Store: ${STORE}\n`);
for (let i = 0; i < SLICE_IDS.length; i++) {
const pid = SLICE_IDS[i];
const shortId = pid.split('/').pop();
const idx = `[${i + 1}/${total}]`;
// 1. Fetch metafields
let meta;
try {
meta = await fetchMetafields(pid);
} catch (e) {
console.log(`${idx} ${shortId} FETCH ERROR: ${e.message}`);
counts.failed++;
failureLog.push(`${shortId}: fetch metafields error: ${e.message}`);
await sleep(400);
continue;
}
if (!meta) {
console.log(`${idx} ${shortId} NOT FOUND in Shopify`);
counts.failed++;
failureLog.push(`${shortId}: product not found`);
await sleep(400);
continue;
}
// Guard: only process Rebel Walls products
if (meta.vendor !== 'Rebel Walls') {
console.log(`${idx} ${shortId} GUARD: vendor="${meta.vendor}" not RW, skip`);
counts.failed++;
failureLog.push(`${shortId}: vendor="${meta.vendor}" not RW`);
await sleep(400);
continue;
}
// 2. Check for URL
if (!meta.rw_product_url) {
console.log(`${idx} ${shortId} NO URL — unknown group (title: ${meta.title?.slice(0, 50)})`);
counts.unknown++;
counts.processed++;
await sleep(300);
continue;
}
// 3. Fetch the RW product page
let pricePerM2 = null;
let fetchErr = null;
try {
const { status, body } = await fetchUrl(meta.rw_product_url);
if (status === 200) {
pricePerM2 = extractPriceFromPage(body, meta.rw_product_url);
if (pricePerM2 === null) {
console.log(`${idx} ${shortId} PRICE EXTRACT FAILED url=${meta.rw_product_url}`);
// Debug: check what dollar amounts are present
const anyDollar = body.match(/\$[0-9]+\.[0-9]+/g);
if (anyDollar) console.log(` Dollar amounts in page: ${[...new Set(anyDollar)].slice(0, 10).join(', ')}`);
}
} else {
fetchErr = `HTTP ${status}`;
console.log(`${idx} ${shortId} HTTP ${status} for ${meta.rw_product_url}`);
}
} catch (e) {
fetchErr = e.message;
console.log(`${idx} ${shortId} FETCH PAGE ERROR: ${e.message}`);
}
// 4. Bucket the price
let group;
if (pricePerM2 !== null) {
group = bucketPrice(pricePerM2);
console.log(`${idx} ${shortId} price=$${pricePerM2.toFixed(2)}/m2 -> ${group} (title: ${meta.title?.slice(0, 40)})`);
} else {
// Can't determine -> leave as-is (C3 default), count unknown
console.log(`${idx} ${shortId} UNKNOWN price (${fetchErr || 'no pattern matched'}) -> leaving as-is`);
counts.unknown++;
counts.processed++;
await sleep(400);
continue;
}
// 5. Write metafields
let writeErrors = [];
try {
writeErrors = await writeMetafields(pid, group);
} catch (e) {
writeErrors = [e.message];
}
if (writeErrors.length > 0) {
console.log(`${idx} ${shortId} WRITE ERROR: ${writeErrors.join('; ')}`);
counts.failed++;
failureLog.push(`${shortId}: write errors: ${writeErrors.join('; ')}`);
} else {
counts[group.toLowerCase()]++;
counts.processed++;
console.log(` -> wrote price_group=${group}${group !== 'C3' ? ` cost_per_m2=${GROUPS[group].cost_per_m2}` : ' (C3, cost unchanged)'}`);
}
await sleep(500); // ~2 req/s budget
}
console.log('\n=== DONE ===');
console.log(`processed=${counts.processed} c2=${counts.c2} c3=${counts.c3} c4=${counts.c4} unknown=${counts.unknown} failed=${counts.failed}`);
if (failureLog.length) {
console.log('FAILURES:');
failureLog.forEach(f => console.log(' - ' + f));
}
return counts;
}
process.on('unhandledRejection', (reason) => {
console.error(`UNHANDLED REJECTION: ${reason && reason.stack ? reason.stack : reason}`);
});
process.on('uncaughtException', (e) => {
console.error(`UNCAUGHT EXCEPTION: ${e.stack || e}`);
});
main().catch(e => { console.error(`FATAL: ${e.message}\n${e.stack}`); process.exit(1); });