`onboarding/sangetsu-lilycolor/lily-image-fetch.cjs`

93 lines
#!/usr/bin/env node
/**
 * Selective Lilycolor swatch fetcher — random-access ONE image out of a remote
 * 100–400MB catalog zip via HTTP Range, using offsets from the zip central directory.
 * Pulls the PATTERN swatch (_C, else _P/_SP) per SKU. JPEGs are STORED (method 0) in
 * these zips, so we write the bytes straight out (deflate path handled just in case).
 *
 * Lands images on the HENRY 2TB drive (internal disk is tight) — idempotent (skips
 * already-fetched). Read-only network; no Shopify/dw_unified/publish.
 *   node lily-image-fetch.cjs [limit]   (default: all manifest SKUs lacking a fetched swatch)
 */
const https = require('https');
const zlib = require('zlib');
const fs = require('fs');
const path = require('path');

const DIR = path.join(__dirname, 'staging');
const MANIFEST = path.join(DIR, 'lilycolor-image-manifest.json');
const OUT_DIR = '/Volumes/Henry/dw-lily-images';
const TAIL = 4 * 1024 * 1024;
const KIND_PREF = { C: 0, P: 1, SP: 2, R: 3 };

function getBuf(url, headers) {
  return new Promise((resolve, reject) => {
    https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0 Chrome/120', ...headers } }, (res) => {
      if (res.statusCode >= 400) { res.resume(); return reject(new Error(`HTTP ${res.statusCode}`)); }
      const chunks = []; res.on('data', (c) => chunks.push(c)); res.on('end', () => resolve(Buffer.concat(chunks)));
    }).on('error', reject);
  });
}

// Parse the central directory tail -> map filename -> {offset, compSize, method}
function parseCentralDir(buf) {
  const map = {};
  for (let i = 0; i + 46 <= buf.length; i++) {
    if (buf.readUInt32LE(i) !== 0x02014b50) continue;
    const method = buf.readUInt16LE(i + 10);
    const compSize = buf.readUInt32LE(i + 20);
    const nameLen = buf.readUInt16LE(i + 28);
    const extraLen = buf.readUInt16LE(i + 30);
    const commentLen = buf.readUInt16LE(i + 32);
    const offset = buf.readUInt32LE(i + 42);
    if (nameLen === 0 || nameLen > 300 || i + 46 + nameLen > buf.length) continue;
    const name = buf.slice(i + 46, i + 46 + nameLen).toString('utf8').split('/').pop();
    if (/\.jpe?g$/i.test(name)) map[name] = { offset, compSize, method };
    i += 46 + nameLen + extraLen + commentLen - 1;
  }
  return map;
}

async function fetchFile(zipUrl, rec) {
  // local header: 30 fixed + nameLen + extraLen, then data (compSize). Range over the lot (+slack for extra).
  const start = rec.offset;
  const end = rec.offset + 30 + 300 + 64 + rec.compSize; // generous slack
  const buf = await getBuf(zipUrl, { Range: `bytes=${start}-${end}` });
  if (buf.readUInt32LE(0) !== 0x04034b50) throw new Error('bad local header');
  const nameLen = buf.readUInt16LE(26), extraLen = buf.readUInt16LE(28);
  const dataStart = 30 + nameLen + extraLen;
  const data = buf.slice(dataStart, dataStart + rec.compSize);
  return rec.method === 8 ? zlib.inflateRawSync(data) : data;
}

(async () => {
  if (!fs.existsSync('/Volumes/Henry')) { console.error('Henry drive not mounted at /Volumes/Henry'); process.exit(1); }
  fs.mkdirSync(OUT_DIR, { recursive: true });
  const manifest = JSON.parse(fs.readFileSync(MANIFEST, 'utf8'));
  const limit = parseInt(process.argv[2] || '999999', 10);

  // group target files by zip so we read each central directory once
  const cdCache = {};
  const targets = [];        // {sku, zipUrl, name}
  for (const [sku, imgs] of Object.entries(manifest)) {
    if (fs.existsSync(path.join(OUT_DIR, `${sku}.jpg`))) continue;       // idempotent
    const best = [...imgs].sort((a, b) => (KIND_PREF[a.kind] ?? 9) - (KIND_PREF[b.kind] ?? 9))[0];
    if (best) targets.push({ sku, zipUrl: best.zipUrl, name: best.name });
    if (targets.length >= limit) break;
  }
  console.error(`fetching ${targets.length} swatches -> ${OUT_DIR}`);
  let ok = 0, fail = 0;
  for (const t of targets) {
    try {
      if (!cdCache[t.zipUrl]) cdCache[t.zipUrl] = parseCentralDir(await getBuf(t.zipUrl, { Range: `bytes=-${TAIL}` }));
      const rec = cdCache[t.zipUrl][t.name];
      if (!rec) { fail++; continue; }
      const bytes = await fetchFile(t.zipUrl, rec);
      fs.writeFileSync(path.join(OUT_DIR, `${t.sku}.jpg`), bytes);
      ok++;
      if (ok % 50 === 0) process.stderr.write(`  ${ok}/${targets.length}\n`);
    } catch (e) { fail++; if (fail <= 5) process.stderr.write(`FAIL ${t.sku}: ${e.message}\n`); }
  }
  console.log(JSON.stringify({ fetched: ok, failed: fail, dir: OUT_DIR }, null, 2));
})().catch((e) => { console.error('FAILED:', e.message); process.exit(1); });