← back to Dw Kravet Hires
Add dupe-media cleanup for TK-12097 concurrent Batch-A run
1278615cdcd11b70821c941cff81e7c2206ae68f · 2026-09-24 11:00:57 -0700 · Steve Abrams
Two apply runs processed Batch A at once; each uploaded its own copy of
the same hi-res image, leaving 249 products with a redundant duplicate.
The tool picks which copy to remove from a live per-product read rather
than from ledger order, refuses when the featured media is not one of the
two uploaded copies, and refuses when the low-res rollback anchor is
missing. Dry-run by default; deletions are ledgered with a re-upload undo.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KXUyzc9vybUz39rhnNJdwY
Files touched
A scripts/cleanup-dupe-media.mjsA scripts/cleanup-dupe-media.test.mjs
Diff
commit 1278615cdcd11b70821c941cff81e7c2206ae68f
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Thu Sep 24 11:00:57 2026 -0700
Add dupe-media cleanup for TK-12097 concurrent Batch-A run
Two apply runs processed Batch A at once; each uploaded its own copy of
the same hi-res image, leaving 249 products with a redundant duplicate.
The tool picks which copy to remove from a live per-product read rather
than from ledger order, refuses when the featured media is not one of the
two uploaded copies, and refuses when the low-res rollback anchor is
missing. Dry-run by default; deletions are ledgered with a re-upload undo.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01KXUyzc9vybUz39rhnNJdwY
---
scripts/cleanup-dupe-media.mjs | 206 ++++++++++++++++++++++++++++++++++++
scripts/cleanup-dupe-media.test.mjs | 47 ++++++++
2 files changed, 253 insertions(+)
diff --git a/scripts/cleanup-dupe-media.mjs b/scripts/cleanup-dupe-media.mjs
new file mode 100644
index 0000000..7e57408
--- /dev/null
+++ b/scripts/cleanup-dupe-media.mjs
@@ -0,0 +1,206 @@
+#!/usr/bin/env node
+/**
+ * cleanup-dupe-media.mjs — TK-12097
+ *
+ * Removes the REDUNDANT duplicate hi-res media created when two apply-hires
+ * runs processed Batch A concurrently (both read the same pre-swap state and
+ * each uploaded its own MediaImage from the identical source URL).
+ *
+ * For each product in the manifest it does a LIVE media read and deletes only
+ * the one uploaded copy that is NOT currently featured. Ledger order is never
+ * trusted to decide which id to delete — misreading a ledger is what produced
+ * this cleanup in the first place.
+ *
+ * Dry-run by default. --live is required to delete anything.
+ *
+ * node scripts/cleanup-dupe-media.mjs --manifest data/tk12097/tk12097-batchA-duplicates.json
+ * node scripts/cleanup-dupe-media.mjs --manifest ... --live
+ * node scripts/cleanup-dupe-media.mjs --rollback --ledger data/tk12097/dupe-cleanup-ledger.jsonl --live
+ */
+import fs from 'fs';
+import path from 'path';
+import os from 'os';
+
+const SHOP = 'designer-laboratory-sandbox';
+const API = '2024-10';
+
+const argv = process.argv.slice(2);
+const has = (f) => argv.includes(f);
+const val = (f, d) => { const i = argv.indexOf(f); return i >= 0 && argv[i + 1] ? argv[i + 1] : d; };
+
+const LIVE = has('--live');
+const ROLLBACK = has('--rollback');
+const TICKET = val('--ticket', 'TK-12097');
+const AGENT = process.env.TK_AGENT || 'yoloforever-master';
+const LIMIT = parseInt(val('--limit', '0'), 10);
+const GAP_MS = parseInt(val('--gap-ms', '700'), 10);
+
+const HERE = path.dirname(new URL(import.meta.url).pathname);
+const PROJ = path.resolve(HERE, '..');
+const MANIFEST = path.resolve(PROJ, val('--manifest', 'data/tk12097/tk12097-batchA-duplicates.json'));
+const LEDGER = path.resolve(PROJ, val('--ledger', 'data/tk12097/dupe-cleanup-ledger.jsonl'));
+const EXEC_LEDGER = path.join(os.homedir(), '.claude/yolo-queue/executed-reversible/ledger.jsonl');
+
+function shopTok() {
+ const env = fs.readFileSync(path.join(os.homedir(), 'Projects/secrets-manager/.env'), 'utf8');
+ const admin = (env.match(/^SHOPIFY_ADMIN_TOKEN=(.+)$/m) || [])[1];
+ const full = (env.match(/^SHOPIFY_FULL_ACCESS_TOKEN=(.+)$/m) || [])[1];
+ const tok = admin || full;
+ if (!tok) { console.error('FATAL: no SHOPIFY_ADMIN_TOKEN / SHOPIFY_FULL_ACCESS_TOKEN in secrets .env'); process.exit(2); }
+ return tok.trim();
+}
+
+const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
+const MAX_RETRIES = parseInt(val('--max-retries', '6'), 10);
+
+async function shopify(query, variables, attempt = 0) {
+ const r = await fetch(`https://${SHOP}.myshopify.com/admin/api/${API}/graphql.json`, {
+ method: 'POST',
+ headers: { 'X-Shopify-Access-Token': shopTok(), 'Content-Type': 'application/json' },
+ body: JSON.stringify({ query, variables }),
+ signal: AbortSignal.timeout(60000),
+ });
+ if (r.status === 429) return backoff('HTTP 429', query, variables, attempt, r.headers.get('retry-after'), null);
+ const j = await r.json();
+ if (j.errors) {
+ const throttled = Array.isArray(j.errors) &&
+ j.errors.some((e) => e?.extensions?.code === 'THROTTLED' || /throttl/i.test(e?.message || ''));
+ if (throttled) return backoff('THROTTLED', query, variables, attempt, null, j.extensions?.cost?.throttleStatus);
+ throw new Error('shopify gql: ' + JSON.stringify(j.errors));
+ }
+ return j.data;
+}
+
+async function backoff(reason, query, variables, attempt, retryAfter, throttleStatus) {
+ if (attempt >= MAX_RETRIES) throw new Error(`shopify gql: ${reason} — exhausted ${MAX_RETRIES} retries`);
+ let waitMs = 0;
+ if (throttleStatus?.currentlyAvailable != null && throttleStatus.restoreRate) {
+ waitMs = Math.ceil((Math.max(0, 50 - throttleStatus.currentlyAvailable) / throttleStatus.restoreRate) * 1000);
+ } else if (retryAfter) waitMs = parseFloat(retryAfter) * 1000 || 0;
+ waitMs = Math.max(waitMs, Math.min(30000, 1000 * 2 ** attempt)) + Math.floor(Math.random() * 500);
+ console.log(` … ${reason}, backing off ${Math.round(waitMs)}ms (retry ${attempt + 1}/${MAX_RETRIES})`);
+ await sleep(waitMs);
+ return shopify(query, variables, attempt + 1);
+}
+
+const Q_MEDIA = `query($id: ID!) {
+ product(id: $id) {
+ id title
+ media(first: 50) { edges { node { id ... on MediaImage { image { url width height } } } } }
+ }
+}`;
+
+const M_DELETE = `mutation($productId: ID!, $mediaIds: [ID!]!) {
+ productDeleteMedia(productId: $productId, mediaIds: $mediaIds) {
+ deletedMediaIds
+ mediaUserErrors { field message }
+ }
+}`;
+
+function appendLedger(file, obj) {
+ const fd = fs.openSync(file, 'a');
+ try { fs.writeSync(fd, JSON.stringify(obj) + '\n'); fs.fsyncSync(fd); } finally { fs.closeSync(fd); }
+}
+
+/**
+ * Decide what to delete for one product, from LIVE state only.
+ * Returns {action:'delete', id} | {action:'skip', why} — never guesses.
+ */
+export function decide(liveMediaIds, featuredId, candidateIds) {
+ const present = candidateIds.filter((id) => liveMediaIds.includes(id));
+ if (present.length === 0) return { action: 'skip', why: 'neither uploaded copy present (already cleaned)' };
+ if (present.length === 1) return { action: 'skip', why: 'only one copy present — nothing redundant' };
+ if (!present.includes(featuredId)) {
+ // Refuse: featured is something we did not upload. Deleting here could
+ // remove the only hi-res, or touch media this run never created.
+ return { action: 'skip', why: `featured ${featuredId} is not one of the uploaded copies — REFUSING` };
+ }
+ const redundant = present.filter((id) => id !== featuredId);
+ if (redundant.length !== 1) return { action: 'skip', why: `expected exactly 1 redundant, got ${redundant.length}` };
+ return { action: 'delete', id: redundant[0] };
+}
+
+async function main() {
+ if (ROLLBACK) return rollback();
+
+ const manifest = JSON.parse(fs.readFileSync(MANIFEST, 'utf8'));
+ const rows = LIMIT > 0 ? manifest.slice(0, LIMIT) : manifest;
+ console.log(`${LIVE ? 'LIVE' : 'DRY-RUN'} — ${rows.length} products from ${path.basename(MANIFEST)}`);
+ if (!LIVE) console.log('(no deletes will be made; pass --live to execute)\n');
+
+ let del = 0, skip = 0, err = 0;
+ for (const [i, row] of rows.entries()) {
+ try {
+ const d = await shopify(Q_MEDIA, { id: row.shopify_id });
+ if (!d?.product) { console.log(` ?? ${row.mfr_sku} — product not found`); err++; continue; }
+ const edges = d.product.media.edges;
+ const liveIds = edges.map((e) => e.node.id);
+ const featuredId = edges[0]?.node?.id;
+
+ const verdict = decide(liveIds, featuredId, row.media_ids);
+ if (verdict.action === 'skip') { console.log(` -- ${row.mfr_sku} skip: ${verdict.why}`); skip++; continue; }
+
+ // Safety: the retained low-res anchor must still be on the product.
+ if (row.old_media_id && !liveIds.includes(row.old_media_id)) {
+ console.log(` -- ${row.mfr_sku} skip: rollback anchor ${row.old_media_id} missing — REFUSING`); skip++; continue;
+ }
+
+ if (!LIVE) { console.log(` ~~ ${row.mfr_sku} would delete ${verdict.id} (featured ${featuredId} kept)`); del++; continue; }
+
+ const res = await shopify(M_DELETE, { productId: row.shopify_id, mediaIds: [verdict.id] });
+ const errs = res?.productDeleteMedia?.mediaUserErrors || [];
+ if (errs.length) { console.log(` !! ${row.mfr_sku} — ${JSON.stringify(errs)}`); err++; continue; }
+ const deleted = res?.productDeleteMedia?.deletedMediaIds || [];
+ if (!deleted.includes(verdict.id)) { console.log(` !! ${row.mfr_sku} — API did not confirm deletion`); err++; continue; }
+
+ appendLedger(LEDGER, {
+ ts: new Date().toISOString(), ticket: TICKET, agent: AGENT,
+ shopify_id: row.shopify_id, mfr_sku: row.mfr_sku,
+ deleted_media_id: verdict.id, kept_featured_id: featuredId,
+ old_media_id: row.old_media_id, src: row.src,
+ undo: `re-upload ${row.src} via stagedUploadsCreate + productCreateMedia on ${row.shopify_id}`,
+ });
+ console.log(` ✓ ${row.mfr_sku} deleted ${verdict.id} (featured ${featuredId} kept)`);
+ del++;
+ } catch (e) {
+ console.log(` !! ${row.mfr_sku} — ${e.message}`); err++;
+ }
+ if ((i + 1) % 25 === 0) console.log(` [${i + 1}/${rows.length}]`);
+ await sleep(GAP_MS);
+ }
+
+ console.log(`\n${LIVE ? 'deleted' : 'would delete'}: ${del} | skipped: ${skip} | errors: ${err}`);
+ if (LIVE && del > 0) {
+ appendLedger(EXEC_LEDGER, {
+ ts: new Date().toISOString(), agent: AGENT, ticket: TICKET,
+ action: `Removed ${del} redundant duplicate hi-res media created by concurrent Batch-A apply runs`,
+ blast_radius: del,
+ undo_cmd: `node scripts/cleanup-dupe-media.mjs --rollback --ledger ${path.relative(PROJ, LEDGER)} --live`,
+ verify: `re-run without --live; every row should report "only one copy present"`,
+ });
+ }
+}
+
+async function rollback() {
+ if (!fs.existsSync(LEDGER)) { console.error(`no ledger at ${LEDGER}`); process.exit(2); }
+ const rows = fs.readFileSync(LEDGER, 'utf8').trim().split('\n').filter(Boolean).map(JSON.parse)
+ .filter((r) => r.ticket === TICKET);
+ console.log(`${LIVE ? 'LIVE' : 'DRY-RUN'} rollback — ${rows.length} deleted media to re-upload`);
+ for (const r of rows) {
+ console.log(` ${LIVE ? 're-upload' : 'would re-upload'} ${r.src} -> ${r.mfr_sku} (${r.shopify_id})`);
+ if (!LIVE) continue;
+ const d = await shopify(`mutation($productId: ID!, $media: [CreateMediaInput!]!) {
+ productCreateMedia(productId: $productId, media: $media) {
+ media { ... on MediaImage { id } } mediaUserErrors { field message } } }`,
+ { productId: r.shopify_id, media: [{ originalSource: r.src, mediaContentType: 'IMAGE', alt: r.mfr_sku }] });
+ const errs = d?.productCreateMedia?.mediaUserErrors || [];
+ console.log(errs.length ? ` !! ${JSON.stringify(errs)}` : ` ✓ restored`);
+ await sleep(GAP_MS);
+ }
+}
+
+// Only run when invoked directly. Importing this module (e.g. from the test
+// file) must never execute a delete-capable main — an import is not a request
+// to act on the live store.
+const invokedDirectly = process.argv[1] && fs.realpathSync(process.argv[1]) === fs.realpathSync(new URL(import.meta.url).pathname);
+if (invokedDirectly) main().catch((e) => { console.error(e); process.exit(1); });
diff --git a/scripts/cleanup-dupe-media.test.mjs b/scripts/cleanup-dupe-media.test.mjs
new file mode 100644
index 0000000..ddfc43a
--- /dev/null
+++ b/scripts/cleanup-dupe-media.test.mjs
@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+// Negative tests for cleanup-dupe-media.mjs `decide()`.
+// A delete-tool that has only been shown its happy path is untested where it matters.
+import { decide } from './cleanup-dupe-media.mjs';
+
+const A = 'gid://shopify/MediaImage/AAA'; // uploaded copy 1
+const B = 'gid://shopify/MediaImage/BBB'; // uploaded copy 2
+const OLD = 'gid://shopify/MediaImage/OLD'; // retained low-res anchor
+const X = 'gid://shopify/MediaImage/XXX'; // something we never uploaded
+
+let pass = 0, fail = 0;
+const t = (name, got, want) => {
+ const ok = JSON.stringify(got) === JSON.stringify(want);
+ console.log(`${ok ? 'ok ' : 'FAIL'} ${name}${ok ? '' : `\n got ${JSON.stringify(got)}\n want ${JSON.stringify(want)}`}`);
+ ok ? pass++ : fail++;
+};
+
+// POSITIVE: both copies present, B featured -> delete A
+t('deletes the non-featured copy',
+ decide([B, A, OLD], B, [A, B]), { action: 'delete', id: A });
+
+t('deletes the non-featured copy (order reversed)',
+ decide([A, B, OLD], A, [A, B]), { action: 'delete', id: B });
+
+// NEGATIVE: the cases that must NOT delete
+t('refuses when featured is a media we never uploaded',
+ decide([X, A, B, OLD], X, [A, B]),
+ { action: 'skip', why: `featured ${X} is not one of the uploaded copies — REFUSING` });
+
+t('skips when only one copy remains (already cleaned)',
+ decide([B, OLD], B, [A, B]), { action: 'skip', why: 'only one copy present — nothing redundant' });
+
+t('skips when neither copy is present',
+ decide([OLD], OLD, [A, B]), { action: 'skip', why: 'neither uploaded copy present (already cleaned)' });
+
+t('refuses an empty product (no media at all)',
+ decide([], undefined, [A, B]), { action: 'skip', why: 'neither uploaded copy present (already cleaned)' });
+
+// The critical one: never propose deleting the featured image itself.
+const d = decide([A, B, OLD], A, [A, B]);
+t('never proposes deleting the featured id', d.id !== A, true);
+
+// And never proposes deleting the low-res rollback anchor.
+t('never proposes deleting the anchor', d.id !== OLD, true);
+
+console.log(`\n${pass} passed, ${fail} failed`);
+process.exit(fail ? 1 : 0);
← 4636e7b auto-data-snapshot: 2026-09-24T10:46:49 (3 data files) — dat
·
back to Dw Kravet Hires
·
auto-data-snapshot: 2026-09-24T11:17:31 (3 data files) — dat 68e7ad7 →