refresh: add Meilisearch 20k-cap truncation alarm (contrarian gate #2)

c1db41b92285d05f2163080040c3d8d242ad9579 · 2026-07-08 12:45:59 -0700 · steve@designerwallcoverings.com

Files touched

M momentum-feed/refresh.mjs

Diff

commit c1db41b92285d05f2163080040c3d8d242ad9579
Author: steve@designerwallcoverings.com <steve@designerwallcoverings.com>
Date:   Wed Jul 8 12:45:59 2026 -0700

    refresh: add Meilisearch 20k-cap truncation alarm (contrarian gate #2)
---
 momentum-feed/refresh.mjs | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/momentum-feed/refresh.mjs b/momentum-feed/refresh.mjs
index 6cd151d..2966cc0 100644
--- a/momentum-feed/refresh.mjs
+++ b/momentum-feed/refresh.mjs
@@ -33,9 +33,11 @@ async function fetchPage(offset, limit = 200) {
 
 async function main() {
   const rows = [];
+  let rawSeen = 0, lastOff = 0;
   for (let off = 0; off < 20000; off += 200) {
     const hits = await fetchPage(off);
     if (!hits.length) break;
+    rawSeen += hits.length; lastOff = off + hits.length;
     for (const h of hits) {
       if (!KEEP.has(h.category_name)) continue;
       if (!h.pattern_name || !h.preferred_color_name) continue;
@@ -55,7 +57,11 @@ async function main() {
     }
     process.stdout.write(`\r  fetched offset ${off}  kept ${rows.length}   `);
   }
-  console.log(`\nFeed kept ${rows.length} Wallcovering+Acoustic colorways.`);
+  console.log(`\nFeed kept ${rows.length} Wallcovering+Acoustic colorways (scanned ${rawSeen} raw records).`);
+  if (lastOff >= 20000 || rawSeen >= 20000)
+    console.warn('⚠️  WARNING: hit Meilisearch maxTotalHits=20000 cap — index may be TRUNCATED; ' +
+      'WC/Acoustic records beyond the first 20k are unreachable via offset paging. ' +
+      'Shard the query (e.g. per product_line) if the true index exceeds 20k.');
   const cats = rows.reduce((a, r) => (a[r.category] = (a[r.category] || 0) + 1, a), {});
   console.log('  by category:', JSON.stringify(cats));

← d793c5d momentum LOCAL enrichment runner (qwen2.5vl:7b, $0): ai_colo · back to Hollywood Import · momentum assign-sku: 2902 unmapped colorways → DWHD-508083.. b81b266 →