← back to Letsbegin
rw-price-group-slice16.js
452 lines
#!/usr/bin/env node
/**
* rw-price-group-slice16.js — Price-group detector + cost-metafield writer
* for SLICE at OFFSET 600, LIMIT 40 of the Rebel Walls mural cohort.
*
* Algorithm:
* 1. For each shopify_id, fetch custom.rw_product_url + custom.price_group.
* 2. If no URL -> count unknown, skip.
* 3. Fetch the RW page, extract base per-m2 price from JSON-LD or price chip.
* 4. Bucket: ~58.10 -> C2, ~76.40 -> C3, ~88.30 -> C4 (nearest).
* 5. Write price_group + (if != C3) cost_per_m2 + material_options + cost_basis.
* 6. Report {processed, c2, c3, c4, unknown, failed}.
*
* SANDBOX ONLY. Rate-limited ~2 req/s.
*/
const https = require('https');
const http = require('http');
const { URL } = require('url');
const fs = require('fs');
function loadEnv(path) {
try {
for (const line of fs.readFileSync(path, 'utf8').split('\n')) {
const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)$/);
if (!m) continue;
let v = m[2].trim();
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
if (process.env[m[1]] == null) process.env[m[1]] = v;
}
} catch { /* ok */ }
}
loadEnv('/Users/macstudio3/Projects/secrets-manager/.env');
const STORE = process.env.SHOPIFY_STORE || 'designer-laboratory-sandbox.myshopify.com';
const TOKEN = process.env.SHOPIFY_ADMIN_TOKEN || process.env.SHOPIFY_PRODUCT_TOKEN;
const API = '/admin/api/2024-10/graphql.json';
if (!TOKEN) { console.error('FATAL: SHOPIFY_ADMIN_TOKEN required'); process.exit(1); }
if (STORE !== 'designer-laboratory-sandbox.myshopify.com') {
console.error(`FATAL: refusing to run against non-sandbox store "${STORE}"`); process.exit(1);
}
// ---- the 40 IDs for slice OFFSET 600 ----
const SLICE_IDS = [
'gid://shopify/Product/7855824502835',
'gid://shopify/Product/7855824437299',
'gid://shopify/Product/7855824535603',
'gid://shopify/Product/7855824568371',
'gid://shopify/Product/7855824601139',
'gid://shopify/Product/7855824633907',
'gid://shopify/Product/7855824699443',
'gid://shopify/Product/7855824764979',
'gid://shopify/Product/7855824732211',
'gid://shopify/Product/7855824797747',
'gid://shopify/Product/7855824830515',
'gid://shopify/Product/7855824863283',
'gid://shopify/Product/7855824896051',
'gid://shopify/Product/7855824928819',
'gid://shopify/Product/7855824961587',
'gid://shopify/Product/7855824994355',
'gid://shopify/Product/7855825027123',
'gid://shopify/Product/7855825059891',
'gid://shopify/Product/7855825125427',
'gid://shopify/Product/7855825158195',
'gid://shopify/Product/7855825190963',
'gid://shopify/Product/7855825223731',
'gid://shopify/Product/7855825289267',
'gid://shopify/Product/7855825322035',
'gid://shopify/Product/7855825354803',
'gid://shopify/Product/7855825387571',
'gid://shopify/Product/7855825420339',
'gid://shopify/Product/7855825453107',
'gid://shopify/Product/7855825485875',
'gid://shopify/Product/7855825518643',
'gid://shopify/Product/7855825551411',
'gid://shopify/Product/7855825584179',
'gid://shopify/Product/7855825616947',
'gid://shopify/Product/7855825911859',
'gid://shopify/Product/7855825977395',
'gid://shopify/Product/7855825748019',
'gid://shopify/Product/7855825813555',
'gid://shopify/Product/7855825846323',
'gid://shopify/Product/7855825879091',
'gid://shopify/Product/7855826042931',
];
// ---- price group data ----
const GROUPS = {
C2: {
cost_per_m2: '33.84',
material_options: JSON.stringify([
{"material":"Non-Woven (Standard)","retail_per_m2":58.10,"cost_per_m2":33.84,"cost_confirmed":false},
{"material":"Peel & Stick","retail_per_m2":69.70,"cost_per_m2":40.60,"cost_confirmed":false},
{"material":"Commercial Grade","retail_per_m2":81.36,"cost_per_m2":47.39,"cost_confirmed":false}
]),
retail: 58.10,
cost_basis: 'C2 price group; Non-Woven RRP $58.10/m2; cost = RRP x 0.5825; P&S/Commercial extrapolated. Sourced from live RW public product page.',
},
C3: {
cost_per_m2: '44.50',
material_options: JSON.stringify([
{"material":"Non-Woven (Standard)","retail_per_m2":76.40,"cost_per_m2":44.50,"cost_confirmed":false},
{"material":"Peel & Stick","retail_per_m2":91.70,"cost_per_m2":53.42,"cost_confirmed":false},
{"material":"Commercial Grade","retail_per_m2":107.00,"cost_per_m2":62.33,"cost_confirmed":false}
]),
retail: 76.40,
cost_basis: 'C3 price group; Non-Woven RRP $76.40/m2; cost = RRP x 0.5825; P&S/Commercial extrapolated. Sourced from live RW public product page.',
},
C4: {
cost_per_m2: '51.44',
material_options: JSON.stringify([
{"material":"Non-Woven (Standard)","retail_per_m2":88.30,"cost_per_m2":51.44,"cost_confirmed":false},
{"material":"Peel & Stick","retail_per_m2":106.00,"cost_per_m2":61.75,"cost_confirmed":false},
{"material":"Commercial Grade","retail_per_m2":123.66,"cost_per_m2":72.03,"cost_confirmed":false}
]),
retail: 88.30,
cost_basis: 'C4 price group; Non-Woven RRP $88.30/m2; cost = RRP x 0.5825; P&S/Commercial extrapolated. Sourced from live RW public product page.',
},
};
function bucketPrice(pricePerM2) {
// Snap to nearest anchor: C2=58.10, C3=76.40, C4=88.30
const anchors = [
{ group: 'C2', anchor: 58.10 },
{ group: 'C3', anchor: 76.40 },
{ group: 'C4', anchor: 88.30 },
];
let best = anchors[0];
let bestDist = Math.abs(pricePerM2 - anchors[0].anchor);
for (const a of anchors) {
const d = Math.abs(pricePerM2 - a.anchor);
if (d < bestDist) { bestDist = d; best = a; }
}
return best.group;
}
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
// ---- Shopify GraphQL ----
function gqlRaw(body) {
return new Promise((resolve, reject) => {
const data = JSON.stringify(body);
const req = https.request({
hostname: STORE, path: API, method: 'POST',
headers: { 'X-Shopify-Access-Token': TOKEN, 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data) },
}, res => { let c = ''; res.on('data', d => c += d); res.on('end', () => { try { resolve(JSON.parse(c)); } catch { resolve({ error: c.slice(0, 300) }); } }); });
req.on('error', reject);
req.setTimeout(60000, () => { req.destroy(); reject(new Error('timeout')); });
req.write(data); req.end();
});
}
async function gql(body, retries = 5) {
for (let attempt = 1; attempt <= retries; attempt++) {
try {
const result = await gqlRaw(body);
const throttled = result?.errors?.some(e => /Throttled/i.test(e.message || ''));
const lowBudget = (result?.extensions?.cost?.throttleStatus?.currentlyAvailable || 9999) < 200;
if (throttled) { await sleep(3000); continue; }
if (lowBudget) await sleep(1500);
return result;
} catch (e) {
if (attempt < retries) { await sleep(attempt * 2000); continue; }
throw e;
}
}
}
const MF_MUTATION = 'mutation metafieldsSet($m: [MetafieldsSetInput!]!) { metafieldsSet(metafields: $m) { metafields { key } userErrors { message field } } }';
async function fetchMetafields(pid) {
const r = await gql({ query: `{ product(id:"${pid}") {
id title vendor
url: metafield(namespace:"custom", key:"rw_product_url"){ value }
pg: metafield(namespace:"custom", key:"price_group"){ value }
cost: metafield(namespace:"custom", key:"cost_per_m2"){ value }
} }` });
const p = r?.data?.product;
if (!p) return null;
return {
id: p.id,
title: p.title,
vendor: p.vendor,
rw_product_url: p.url?.value ?? null,
price_group: p.pg?.value ?? null,
cost_per_m2: p.cost?.value ?? null,
};
}
// ---- fetch RW product page and extract base price ----
function fetchPageUrl(urlStr, redirects = 5) {
return new Promise((resolve, reject) => {
try {
const u = new URL(urlStr);
const lib = u.protocol === 'https:' ? https : http;
let resolved = false;
const req = lib.request({
hostname: u.hostname,
path: u.pathname + u.search,
method: 'GET',
headers: {
'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/124 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,*/*;q=0.9',
'Accept-Language': 'en-US,en;q=0.9',
'Connection': 'close',
},
}, res => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && redirects > 0) {
res.resume();
req.destroy();
let loc = res.headers.location;
if (loc.startsWith('/')) loc = u.origin + loc;
if (!resolved) { resolved = true; resolve(fetchPageUrl(loc, redirects - 1)); }
return;
}
let c = '';
let truncated = false;
res.setEncoding('utf8');
res.on('data', d => {
if (truncated) return;
c += d;
if (c.length > 400000) {
// We have enough to extract prices — stop consuming but don't destroy
truncated = true;
res.destroy();
if (!resolved) { resolved = true; resolve({ status: res.statusCode, body: c }); }
}
});
res.on('end', () => { if (!resolved) { resolved = true; resolve({ status: res.statusCode, body: c }); } });
res.on('error', () => { if (!resolved) { resolved = true; resolve({ status: res.statusCode, body: c }); } });
});
req.on('error', e => { if (!resolved) { resolved = true; reject(e); } });
req.setTimeout(30000, () => { req.destroy(); if (!resolved) { resolved = true; reject(new Error('timeout fetching ' + urlStr)); } });
req.end();
} catch (e) { reject(e); }
});
}
function extractPriceFromPage(html) {
// Strategy 1: JSON-LD with offers.price
const jsonldMatches = [...html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi)];
for (const m of jsonldMatches) {
try {
const obj = JSON.parse(m[1]);
const items = Array.isArray(obj) ? obj : [obj];
for (const item of items) {
const price = item?.offers?.price || item?.offers?.[0]?.price;
if (price && !isNaN(parseFloat(price))) {
const p = parseFloat(price);
// RW prices: $5-10/sqft or $55-100/m2
if (p >= 2 && p <= 20) return p * 10.7639; // per sqft -> m2
if (p >= 40 && p <= 200) return p; // already per m2
}
}
} catch { /* ignore */ }
}
// Strategy 2: "From $X / sq ft" patterns
const sqftPatterns = [
/From\s+\$([0-9]+\.?[0-9]*)\s*\/\s*sq\.?\s*ft/i,
/\$([0-9]+\.?[0-9]*)\s*\/\s*sq\.?\s*ft/i,
/From\s+USD\s*([0-9]+\.?[0-9]*)\s*\/\s*sq/i,
];
for (const pat of sqftPatterns) {
const m = html.match(pat);
if (m) {
const p = parseFloat(m[1]);
if (p >= 2 && p <= 20) return p * 10.7639;
}
}
// Strategy 3: per m2 patterns
const m2Patterns = [
/From\s+\$([0-9]+\.?[0-9]*)\s*\/\s*m[²2]/i,
/\$([0-9]+\.?[0-9]*)\s*\/\s*m[²2]/i,
/From\s+USD\s*([0-9]+\.?[0-9]*)\s*\/\s*m[²2]/i,
];
for (const pat of m2Patterns) {
const m = html.match(pat);
if (m) {
const p = parseFloat(m[1]);
if (p >= 40 && p <= 200) return p;
}
}
// Strategy 4: data attributes or JS variables with price
const priceAttrPatterns = [
/"price"\s*:\s*([0-9]+\.?[0-9]*)/,
/data-price="([0-9]+\.?[0-9]*)"/,
/price_per_sqft['":\s]+([0-9]+\.?[0-9]*)/i,
/pricePerSqFt['":\s]+([0-9]+\.?[0-9]*)/i,
];
for (const pat of priceAttrPatterns) {
const m = html.match(pat);
if (m) {
const p = parseFloat(m[1]);
if (p >= 2 && p <= 20) return p * 10.7639;
if (p >= 40 && p <= 200) return p;
}
}
// Strategy 5: look for standalone price-like numbers near pricing keywords
const priceSectionMatch = html.match(/(?:From|Starting|Price|Cost)[^$]*\$([0-9]+\.[0-9]+)/i);
if (priceSectionMatch) {
const p = parseFloat(priceSectionMatch[1]);
if (p >= 2 && p <= 20) return p * 10.7639;
if (p >= 40 && p <= 200) return p;
}
return null;
}
async function writeMetafields(pid, group) {
const g = GROUPS[group];
const mf = [
{ ownerId: pid, namespace: 'custom', key: 'price_group', value: group, type: 'single_line_text_field' },
{ ownerId: pid, namespace: 'custom', key: 'cost_basis', value: g.cost_basis, type: 'multi_line_text_field' },
];
// For C2 and C4, overwrite cost_per_m2 and material_options
if (group !== 'C3') {
mf.push({ ownerId: pid, namespace: 'custom', key: 'cost_per_m2', value: g.cost_per_m2, type: 'number_decimal' });
mf.push({ ownerId: pid, namespace: 'custom', key: 'material_options', value: g.material_options, type: 'json' });
}
const r = await gql({ query: MF_MUTATION, variables: { m: mf } });
const errs = r?.data?.metafieldsSet?.userErrors || [];
return errs.map(e => `${(e.field || []).join('.')}: ${e.message}`);
}
async function main() {
const counts = { processed: 0, c2: 0, c3: 0, c4: 0, unknown: 0, failed: 0 };
const failureLog = [];
const total = SLICE_IDS.length;
console.log(`[rw-price-group-slice16] Processing ${total} products (slice OFFSET 600)`);
console.log(`Store: ${STORE}\n`);
for (let i = 0; i < SLICE_IDS.length; i++) {
const pid = SLICE_IDS[i];
const idx = `[${i + 1}/${total}]`;
const shortId = pid.split('/').pop();
// 1. Fetch metafields
let meta;
try {
meta = await fetchMetafields(pid);
} catch (e) {
console.log(`${idx} ${shortId} FETCH ERROR: ${e.message}`);
counts.failed++;
failureLog.push(`${shortId}: fetch metafields error: ${e.message}`);
await sleep(400);
continue;
}
if (!meta) {
console.log(`${idx} ${shortId} NOT FOUND in Shopify`);
counts.failed++;
failureLog.push(`${shortId}: product not found`);
await sleep(400);
continue;
}
// Guard: only process Rebel Walls products
if (meta.vendor !== 'Rebel Walls') {
console.log(`${idx} ${shortId} GUARD: vendor="${meta.vendor}" not RW, skip`);
counts.failed++;
failureLog.push(`${shortId}: vendor="${meta.vendor}" not RW`);
await sleep(400);
continue;
}
// 2. Check for URL
if (!meta.rw_product_url) {
console.log(`${idx} ${shortId} NO URL — unknown group (title: ${meta.title?.slice(0, 50)})`);
counts.unknown++;
counts.processed++;
await sleep(300);
continue;
}
// 3. Fetch the RW product page
let pricePerM2 = null;
let fetchErr = null;
try {
const { status, body } = await fetchPageUrl(meta.rw_product_url);
if (status === 200) {
pricePerM2 = extractPriceFromPage(body);
if (pricePerM2 === null) {
console.log(`${idx} ${shortId} PRICE EXTRACT FAILED url=${meta.rw_product_url}`);
const snippet = body.slice(0, 5000);
const anyDollar = snippet.match(/\$[0-9]+\.[0-9]+/g);
if (anyDollar) console.log(` Dollar amounts in first 5k: ${anyDollar.slice(0, 10).join(', ')}`);
}
} else {
fetchErr = `HTTP ${status}`;
console.log(`${idx} ${shortId} HTTP ${status} for ${meta.rw_product_url}`);
}
} catch (e) {
fetchErr = e.message;
console.log(`${idx} ${shortId} FETCH PAGE ERROR: ${e.message}`);
}
// 4. Bucket the price
let group;
if (pricePerM2 !== null) {
group = bucketPrice(pricePerM2);
console.log(`${idx} ${shortId} price=$${pricePerM2.toFixed(2)}/m2 -> ${group} (title: ${meta.title?.slice(0, 40)})`);
} else {
// Can't determine -> leave as-is (C3 default), count unknown
console.log(`${idx} ${shortId} UNKNOWN price (err: ${fetchErr || 'no pattern matched'}) -> leaving C3 default`);
counts.unknown++;
counts.processed++;
await sleep(400);
continue;
}
// 5. Write metafields
let writeErrors = [];
try {
writeErrors = await writeMetafields(pid, group);
} catch (e) {
writeErrors = [e.message];
}
if (writeErrors.length > 0) {
console.log(`${idx} ${shortId} WRITE ERROR: ${writeErrors.join('; ')}`);
counts.failed++;
failureLog.push(`${shortId}: write errors: ${writeErrors.join('; ')}`);
} else {
counts[group.toLowerCase()]++;
counts.processed++;
console.log(` -> wrote price_group=${group}${group !== 'C3' ? ` cost_per_m2=${GROUPS[group].cost_per_m2}` : ' (C3, cost unchanged)'}`);
}
await sleep(500); // ~2 req/s budget
}
console.log('\n=== DONE ===');
console.log(`processed=${counts.processed} c2=${counts.c2} c3=${counts.c3} c4=${counts.c4} unknown=${counts.unknown} failed=${counts.failed}`);
if (failureLog.length) {
console.log('FAILURES:');
failureLog.forEach(f => console.log(' - ' + f));
}
// Emit a final JSON line for easy parsing
console.log('\nJSON_RESULT:' + JSON.stringify({ ...counts, notes: failureLog.join('; ') || 'none' }));
return counts;
}
main().catch(e => { console.error(`FATAL: ${e.message}\n${e.stack}`); process.exit(1); });