ingest: partner-scoped CJ sweeps (partnerIds) + TWOPAGES program joined 2026-08-02 — keyword search can't reach niche joined advertisers; feed indexes 24-48h post-join, re-runs pick it up

96f59a1b907b61fa53c6af11f0c0a3c12fe39172 · 2026-08-02 23:48:26 -0700 · Steve Abrams
Files touched

M lib/adapters/cj.js
M scripts/ingest-cj-catalog.js
Diff

commit 96f59a1b907b61fa53c6af11f0c0a3c12fe39172
Author: Steve Abrams <steve@designerwallcoverings.com>
Date:   Sun Aug 2 23:48:26 2026 -0700

    ingest: partner-scoped CJ sweeps (partnerIds) + TWOPAGES program joined 2026-08-02 — keyword search can't reach niche joined advertisers; feed indexes 24-48h post-join, re-runs pick it up
---
 lib/adapters/cj.js           | 15 ++++++++++++---
 scripts/ingest-cj-catalog.js | 26 +++++++++++++++++++++-----
 2 files changed, 33 insertions(+), 8 deletions(-)

diff --git a/lib/adapters/cj.js b/lib/adapters/cj.js
index 9b3d05f..ce3e7c5 100644
--- a/lib/adapters/cj.js
+++ b/lib/adapters/cj.js
@@ -98,6 +98,12 @@ async function fetch_(env, opts) {
   const { token, companyId, websiteId } = creds(env);
   const limit = Math.min(Math.max(parseInt(opts.limit, 10) || 50, 1), MAX_LIMIT);
   const keywords = (opts && opts.keywords) || env.CJ_KEYWORDS || DEFAULT_KEYWORDS;
+  // partnerIds scopes the sweep to specific JOINED advertisers — the only way to
+  // pull a niche advertiser's full datafeed when generic keywords are dominated
+  // by big unjoined advertisers at the page cap (verified ACCEPTED 2026-08-02).
+  const partnerIds = Array.isArray(opts.partnerIds) && opts.partnerIds.length
+    ? opts.partnerIds.map(String)
+    : null;
 
   // GraphQL products query — VERIFIED against the live CJ schema (2026-08-01):
   //  - companyId: ID!, keywords: String (single, NOT a list), limit: Int
@@ -105,8 +111,8 @@ async function fetch_(env, opts) {
   //    is our website id (CJ_WEBSITE_ID). `link` is the advertiser's raw URL.
   //  - `availability`/`category` are not on the base Product type; dropped.
   const query = `
-    query Products($companyId: ID!, $keywords: [String!], $limit: Int, $pid: ID!) {
-      products(companyId: $companyId, keywords: $keywords, limit: $limit) {
+    query Products($companyId: ID!, $keywords: [String!], $partnerIds: [ID!], $limit: Int, $pid: ID!) {
+      products(companyId: $companyId, keywords: $keywords, partnerIds: $partnerIds, limit: $limit) {
         totalCount
         resultList {
           id
@@ -126,7 +132,10 @@ async function fetch_(env, opts) {
 
   const variables = {
     companyId: String(companyId),
-    keywords: [String(keywords)],
+    // A partner-scoped sweep wants the WHOLE feed — only send keywords when the
+    // caller gave some (or when we're not partner-scoped, to bound the query).
+    keywords: partnerIds && !opts.keywords ? null : [String(keywords)],
+    partnerIds,
     limit,
     pid: String(websiteId),
   };
diff --git a/scripts/ingest-cj-catalog.js b/scripts/ingest-cj-catalog.js
index 6dbcc0f..141372c 100644
--- a/scripts/ingest-cj-catalog.js
+++ b/scripts/ingest-cj-catalog.js
@@ -28,6 +28,15 @@ const CATEGORIES = [
   'robot vacuum', 'cordless vacuum', 'honiture',
 ];
 
+// Joined advertisers swept by partnerIds (whole datafeed, always tracked) —
+// keyword search can't reach these: 'curtains' is saturated by unjoined giants
+// (SHEIN/Temu/Wayfair) and the brand keyword only surfaces resellers.
+// TWOPAGES joined 2026-08-02 (CJ welcome email); feed may lag the join by
+// 24-48h, so a zero sweep is expected until CJ indexes it — re-runs pick it up.
+const PARTNERS = [
+  { id: '7835575', name: 'twopages' }, // custom curtains, 10% commission
+];
+
 async function main() {
   await db.query('SELECT 1');
   if (!cj.enabled(process.env)) { console.error('CJ not configured (need CJ_TOKEN/CJ_COMPANY_ID/CJ_WEBSITE_ID)'); process.exit(1); }
@@ -35,11 +44,13 @@ async function main() {
   const only = (arg('only', '') || '').split(',').filter(Boolean);
   const cats = only.length ? CATEGORIES.filter(c => only.some(o => c.includes(o))) : CATEGORIES;
 
+  const partners = only.length ? PARTNERS.filter(p => only.some(o => p.name.includes(o))) : PARTNERS;
+
   let total = 0, withImg = 0, tracked = 0;
-  for (const cat of cats) {
+  const sweep = async (label, fetchOpts) => {
     let raws = [];
-    try { raws = await cj.fetch(process.env, { limit: per, keywords: cat }); }
-    catch (e) { console.log(`  ${cat}: ERROR ${e.message}`); continue; }
+    try { raws = await cj.fetch(process.env, fetchOpts); }
+    catch (e) { console.log(`  ${label}: ERROR ${e.message}`); return; }
     let ok = 0;
     for (const raw of raws) {
       if (!raw.image_url) continue;           // Room Builder needs images
@@ -49,9 +60,14 @@ async function main() {
       if (!norm) continue;
       try { await db.query(UPSERT_SQL, upsertParams(norm)); ok++; total++; } catch (_) {}
     }
-    console.log(`  ${cat.padEnd(24)} -> ${ok} upserted (of ${raws.length})`);
+    console.log(`  ${label.padEnd(24)} -> ${ok} upserted (of ${raws.length})`);
     await sleep(400); // pace the API
-  }
+  };
+
+  for (const cat of cats) await sweep(cat, { limit: per, keywords: cat });
+  // Partner-scoped sweeps pull the advertiser's whole feed (up to the cap) so a
+  // freshly-joined niche program lands even when keywords can't reach it.
+  for (const p of partners) await sweep(`partner:${p.name}`, { limit: Math.max(per, 500), partnerIds: [p.id] });
   const { rows } = await db.query(`SELECT count(*) n, count(image_url) img FROM products WHERE network='cj'`);
   console.log(`\nDONE. this run: ${total} upserted, ${tracked} with tracked links.`);
   console.log(`catalog now: ${rows[0].n} CJ products (${rows[0].img} with images).`);
← b8892c3 ingest: add robot/cordless vacuum + honiture brand sweep — H · back to Interiordesignershowroom · fix(render): match bigint product ids type-safe — /api/rende 713ee0d →