← back to Letsbegin
rw-price-group-slice2.js
406 lines
#!/usr/bin/env node
/**
* rw-price-group-slice2.js — Price-group detector + cost-metafield writer
* for SLICE 2 (OFFSET 40, LIMIT 40) of the Rebel Walls mural cohort.
*
* Algorithm:
* 1. For each shopify_id, fetch custom.rw_product_url + custom.price_group.
* 2. If no URL -> count unknown, skip.
* 3. curl the RW page, extract base per-m2 price from JSON-LD or price chip.
* 4. Bucket: ~58.10 -> C2, ~76.40 -> C3, ~88.30 -> C4 (nearest).
* 5. Write price_group + (if != C3) cost_per_m2 + material_options + cost_basis.
* 6. Report {processed, c2, c3, c4, unknown, failed}.
*
* SANDBOX ONLY. Rate-limited ~2 req/s.
*/
const https = require('https');
// ---- env ----
const fs = require('fs');
function loadEnv(path) {
try {
for (const line of fs.readFileSync(path, 'utf8').split('\n')) {
const m = line.match(/^\s*([A-Z0-9_]+)\s*=\s*(.*)$/);
if (!m) continue;
let v = m[2].trim();
if ((v.startsWith('"') && v.endsWith('"')) || (v.startsWith("'") && v.endsWith("'"))) v = v.slice(1, -1);
if (process.env[m[1]] == null) process.env[m[1]] = v;
}
} catch { /* ok */ }
}
loadEnv('/Users/macstudio3/Projects/secrets-manager/.env');
const STORE = process.env.SHOPIFY_STORE || 'designer-laboratory-sandbox.myshopify.com';
const TOKEN = process.env.SHOPIFY_ADMIN_TOKEN || process.env.SHOPIFY_PRODUCT_TOKEN;
const API = '/admin/api/2024-10/graphql.json';
if (!TOKEN) { console.error('FATAL: SHOPIFY_ADMIN_TOKEN required'); process.exit(1); }
if (STORE !== 'designer-laboratory-sandbox.myshopify.com') {
console.error(`FATAL: refusing to run against non-sandbox store "${STORE}"`); process.exit(1);
}
// ---- the 40 IDs for slice 2 ----
const SLICE_IDS = [
'gid://shopify/Product/7851165712435',
'gid://shopify/Product/7851165745203',
'gid://shopify/Product/7851165777971',
'gid://shopify/Product/7851165810739',
'gid://shopify/Product/7851165843507',
'gid://shopify/Product/7851165876275',
'gid://shopify/Product/7851165909043',
'gid://shopify/Product/7851165941811',
'gid://shopify/Product/7851165974579',
'gid://shopify/Product/7851166007347',
'gid://shopify/Product/7851166040115',
'gid://shopify/Product/7851166072883',
'gid://shopify/Product/7851164762163',
'gid://shopify/Product/7851166105651',
'gid://shopify/Product/7851166138419',
'gid://shopify/Product/7851166171187',
'gid://shopify/Product/7851166203955',
'gid://shopify/Product/7851166236723',
'gid://shopify/Product/7851166269491',
'gid://shopify/Product/7851166302259',
'gid://shopify/Product/7851166335027',
'gid://shopify/Product/7851166367795',
'gid://shopify/Product/7851166400563',
'gid://shopify/Product/7851166433331',
'gid://shopify/Product/7851166466099',
'gid://shopify/Product/7851166498867',
'gid://shopify/Product/7851166531635',
'gid://shopify/Product/7851166564403',
'gid://shopify/Product/7851166597171',
'gid://shopify/Product/7851166629939',
'gid://shopify/Product/7851166662707',
'gid://shopify/Product/7851166695475',
'gid://shopify/Product/7851166728243',
'gid://shopify/Product/7851166761011',
'gid://shopify/Product/7851166793779',
'gid://shopify/Product/7851166826547',
'gid://shopify/Product/7851166859315',
'gid://shopify/Product/7851166892083',
'gid://shopify/Product/7851166924851',
'gid://shopify/Product/7851166957619',
];
// ---- price group data ----
const GROUPS = {
C2: {
cost_per_m2: '33.84',
material_options: JSON.stringify([
{"material":"Non-Woven (Standard)","retail_per_m2":58.10,"cost_per_m2":33.84,"cost_confirmed":false},
{"material":"Peel & Stick","retail_per_m2":69.70,"cost_per_m2":40.60,"cost_confirmed":false},
{"material":"Commercial Grade","retail_per_m2":81.36,"cost_per_m2":47.39,"cost_confirmed":false}
]),
retail: 58.10,
cost_basis: 'C2 price group; Non-Woven RRP $58.10/m2; cost = RRP x 0.5825; P&S/Commercial extrapolated. Sourced from live RW public product page.',
},
C3: {
cost_per_m2: '44.50',
material_options: JSON.stringify([
{"material":"Non-Woven (Standard)","retail_per_m2":76.40,"cost_per_m2":44.50,"cost_confirmed":false},
{"material":"Peel & Stick","retail_per_m2":91.70,"cost_per_m2":53.42,"cost_confirmed":false},
{"material":"Commercial Grade","retail_per_m2":107.00,"cost_per_m2":62.33,"cost_confirmed":false}
]),
retail: 76.40,
cost_basis: 'C3 price group; Non-Woven RRP $76.40/m2; cost = RRP x 0.5825; P&S/Commercial extrapolated. Sourced from live RW public product page.',
},
C4: {
cost_per_m2: '51.44',
material_options: JSON.stringify([
{"material":"Non-Woven (Standard)","retail_per_m2":88.30,"cost_per_m2":51.44,"cost_confirmed":false},
{"material":"Peel & Stick","retail_per_m2":106.00,"cost_per_m2":61.75,"cost_confirmed":false},
{"material":"Commercial Grade","retail_per_m2":123.66,"cost_per_m2":72.03,"cost_confirmed":false}
]),
retail: 88.30,
cost_basis: 'C4 price group; Non-Woven RRP $88.30/m2; cost = RRP x 0.5825; P&S/Commercial extrapolated. Sourced from live RW public product page.',
},
};
function bucketPrice(pricePerM2) {
// Snap to nearest anchor: C2=58.10, C3=76.40, C4=88.30
const anchors = [
{ group: 'C2', anchor: 58.10 },
{ group: 'C3', anchor: 76.40 },
{ group: 'C4', anchor: 88.30 },
];
let best = anchors[0];
let bestDist = Math.abs(pricePerM2 - anchors[0].anchor);
for (const a of anchors) {
const d = Math.abs(pricePerM2 - a.anchor);
if (d < bestDist) { bestDist = d; best = a; }
}
return best.group;
}
function sleep(ms) { return new Promise(r => setTimeout(r, ms)); }
// ---- Shopify GraphQL ----
function gqlRaw(body) {
return new Promise((resolve, reject) => {
const data = JSON.stringify(body);
const req = https.request({
hostname: STORE, path: API, method: 'POST',
headers: { 'X-Shopify-Access-Token': TOKEN, 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(data) },
}, res => { let c = ''; res.on('data', d => c += d); res.on('end', () => { try { resolve(JSON.parse(c)); } catch { resolve({ error: c.slice(0, 300) }); } }); });
req.on('error', reject);
req.setTimeout(60000, () => { req.destroy(); reject(new Error('timeout')); });
req.write(data); req.end();
});
}
async function gql(body, retries = 5) {
for (let attempt = 1; attempt <= retries; attempt++) {
try {
const result = await gqlRaw(body);
const throttled = result?.errors?.some(e => /Throttled/i.test(e.message || ''));
const lowBudget = (result?.extensions?.cost?.throttleStatus?.currentlyAvailable || 9999) < 200;
if (throttled) { await sleep(3000); continue; }
if (lowBudget) await sleep(1500);
return result;
} catch (e) {
if (attempt < retries) { await sleep(attempt * 2000); continue; }
throw e;
}
}
}
const MF_MUTATION = 'mutation metafieldsSet($m: [MetafieldsSetInput!]!) { metafieldsSet(metafields: $m) { metafields { key } userErrors { message field } } }';
async function fetchMetafields(pid) {
const r = await gql({ query: `{ product(id:"${pid}") {
id title vendor
url: metafield(namespace:"custom", key:"rw_product_url"){ value }
pg: metafield(namespace:"custom", key:"price_group"){ value }
cost: metafield(namespace:"custom", key:"cost_per_m2"){ value }
} }` });
const p = r?.data?.product;
if (!p) return null;
return {
id: p.id,
title: p.title,
vendor: p.vendor,
rw_product_url: p.url?.value ?? null,
price_group: p.pg?.value ?? null,
cost_per_m2: p.cost?.value ?? null,
};
}
// ---- fetch RW product page and extract base price ----
// Use curl subprocess — Node https.request hangs on rebelwalls.com due to HTTPS/Keep-Alive behavior
const { execFile } = require('child_process');
function fetchUrl(urlStr) {
return new Promise((resolve, reject) => {
execFile('curl', [
'-s', '-L',
'--max-time', '15',
'-A', 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 Chrome/124 Safari/537.36',
'-H', 'Accept: text/html,application/xhtml+xml,*/*;q=0.9',
'-H', 'Accept-Language: en-US,en;q=0.9',
urlStr,
], { maxBuffer: 2 * 1024 * 1024 }, (err, stdout, stderr) => {
if (err) { reject(new Error(err.message || 'curl failed')); return; }
resolve({ status: 200, body: stdout });
});
});
}
function extractPriceFromPage(html) {
// Strategy 1: JSON-LD with offers.price (handles both number and string-encoded prices)
const jsonldMatches = [...html.matchAll(/<script[^>]*type="application\/ld\+json"[^>]*>([\s\S]*?)<\/script>/gi)];
for (const m of jsonldMatches) {
try {
const obj = JSON.parse(m[1]);
const items = Array.isArray(obj) ? obj : [obj];
for (const item of items) {
const offers = item?.offers;
if (!offers) continue;
const offerList = Array.isArray(offers) ? offers : [offers];
for (const o of offerList) {
const price = o?.price;
if (price == null) continue;
const raw = String(price).replace(/[^0-9.]/g, '');
if (!raw) continue;
const p = parseFloat(raw);
if (p >= 40 && p <= 200) return p; // USD per m2
if (p >= 2 && p <= 20) return p * 10.7639; // USD per sqft -> m2
}
}
} catch { /* ignore */ }
}
// Strategy 2: bare "price": "76.4" numeric-string in JSON on the page
const bareM = html.match(/"price"\s*:\s*"([0-9]+\.?[0-9]*)"/);
if (bareM) {
const p = parseFloat(bareM[1]);
if (p >= 40 && p <= 200) return p;
if (p >= 2 && p <= 20) return p * 10.7639;
}
// Strategy 3: "$X / sq ft" patterns
const sqftPatterns = [
/From\s+\$([0-9]+\.?[0-9]*)\s*\/\s*sq\.?\s*ft/i,
/\$([0-9]+\.?[0-9]*)\s*\/\s*sq\.?\s*ft/i,
];
for (const pat of sqftPatterns) {
const m = html.match(pat);
if (m) {
const p = parseFloat(m[1]);
if (p >= 2 && p <= 20) return p * 10.7639;
}
}
// Strategy 4: "$X / m2" patterns
const m2Patterns = [
/From\s+\$([0-9]+\.?[0-9]*)\s*\/\s*m[²2]/i,
/\$([0-9]+\.?[0-9]*)\s*\/\s*m[²2]/i,
];
for (const pat of m2Patterns) {
const m = html.match(pat);
if (m) {
const p = parseFloat(m[1]);
if (p >= 40 && p <= 200) return p;
}
}
return null;
}
async function writeMetafields(pid, group) {
const g = GROUPS[group];
const mf = [
{ ownerId: pid, namespace: 'custom', key: 'price_group', value: group, type: 'single_line_text_field' },
{ ownerId: pid, namespace: 'custom', key: 'cost_basis', value: g.cost_basis, type: 'multi_line_text_field' },
];
// For C2 and C4, overwrite cost_per_m2 and material_options
if (group !== 'C3') {
mf.push({ ownerId: pid, namespace: 'custom', key: 'cost_per_m2', value: g.cost_per_m2, type: 'number_decimal' });
mf.push({ ownerId: pid, namespace: 'custom', key: 'material_options', value: g.material_options, type: 'json' });
}
const r = await gql({ query: MF_MUTATION, variables: { m: mf } });
const errs = r?.data?.metafieldsSet?.userErrors || [];
return errs.map(e => `${(e.field || []).join('.')}: ${e.message}`);
}
async function main() {
const counts = { processed: 0, c2: 0, c3: 0, c4: 0, unknown: 0, failed: 0 };
const failureLog = [];
const total = SLICE_IDS.length;
console.log(`[rw-price-group-slice2] Processing ${total} products (slice OFFSET 40)`);
console.log(`Store: ${STORE}\n`);
for (let i = 0; i < SLICE_IDS.length; i++) {
const pid = SLICE_IDS[i];
const idx = `[${i + 1}/${total}]`;
// 1. Fetch metafields
let meta;
try {
meta = await fetchMetafields(pid);
} catch (e) {
console.log(`${idx} ${pid} FETCH ERROR: ${e.message}`);
counts.failed++;
failureLog.push(`${pid}: fetch metafields error: ${e.message}`);
await sleep(400);
continue;
}
if (!meta) {
console.log(`${idx} ${pid} NOT FOUND in Shopify`);
counts.failed++;
failureLog.push(`${pid}: product not found`);
await sleep(400);
continue;
}
// Guard: only process Rebel Walls products
if (meta.vendor !== 'Rebel Walls') {
console.log(`${idx} ${pid} GUARD: vendor="${meta.vendor}" not RW, skip`);
counts.failed++;
failureLog.push(`${pid}: vendor="${meta.vendor}" not RW`);
await sleep(400);
continue;
}
// 2. Check for URL
if (!meta.rw_product_url) {
console.log(`${idx} ${pid} NO URL — unknown group (title: ${meta.title?.slice(0, 50)})`);
counts.unknown++;
counts.processed++;
await sleep(300);
continue;
}
// 3. Fetch the RW product page
let pricePerM2 = null;
let fetchErr = null;
try {
const { status, body } = await fetchUrl(meta.rw_product_url);
if (status === 200) {
pricePerM2 = extractPriceFromPage(body);
if (pricePerM2 === null) {
// Try harder: log a snippet for debugging
console.log(`${idx} ${pid} PRICE EXTRACT FAILED (status=${status}) url=${meta.rw_product_url}`);
// Check for any dollar amounts in the page near price context
const snippet = body.slice(0, 5000);
const anyDollar = snippet.match(/\$[0-9]+\.[0-9]+/g);
if (anyDollar) console.log(` Dollar amounts found in first 5k: ${anyDollar.slice(0, 10).join(', ')}`);
}
} else {
fetchErr = `HTTP ${status}`;
console.log(`${idx} ${pid} HTTP ${status} for ${meta.rw_product_url}`);
}
} catch (e) {
fetchErr = e.message;
console.log(`${idx} ${pid} FETCH PAGE ERROR: ${e.message}`);
}
// 4. Bucket the price
let group;
if (pricePerM2 !== null) {
group = bucketPrice(pricePerM2);
console.log(`${idx} ${pid} price=$${pricePerM2.toFixed(2)}/m2 -> ${group} (title: ${meta.title?.slice(0, 40)})`);
} else {
// Can't determine -> leave as-is (C3 default), count unknown
console.log(`${idx} ${pid} UNKNOWN price (err: ${fetchErr || 'no pattern matched'}) -> leaving C3 default`);
counts.unknown++;
counts.processed++;
await sleep(400);
continue;
}
// 5. Write metafields
let writeErrors = [];
try {
writeErrors = await writeMetafields(pid, group);
} catch (e) {
writeErrors = [e.message];
}
if (writeErrors.length > 0) {
console.log(`${idx} ${pid} WRITE ERROR: ${writeErrors.join('; ')}`);
counts.failed++;
failureLog.push(`${pid}: write errors: ${writeErrors.join('; ')}`);
} else {
counts[group.toLowerCase()]++;
counts.processed++;
console.log(` -> wrote price_group=${group}${group !== 'C3' ? ` cost_per_m2=${GROUPS[group].cost_per_m2}` : ' (C3, cost unchanged)'}`);
}
await sleep(500); // ~2 req/s budget (one fetch + one write per cycle)
}
console.log('\n=== DONE ===');
console.log(`processed=${counts.processed} c2=${counts.c2} c3=${counts.c3} c4=${counts.c4} unknown=${counts.unknown} failed=${counts.failed}`);
if (failureLog.length) {
console.log('FAILURES:');
failureLog.forEach(f => console.log(' - ' + f));
}
return counts;
}
main().catch(e => { console.error(`FATAL: ${e.message}\n${e.stack}`); process.exit(1); });