← back to Homesonspec

Pipeline: FORCE_REEXTRACT env to re-extract already-snapshotted pages (backfills new fields like images onto unchanged-source homes) + durable grant note

ba895b72898e25c7ea104a3ad036e5abcfab8644 · 2026-07-22 18:55:10 -0700 · Steve

Files touched

• M apps/workers/src/pipeline.ts

Diff

commit ba895b72898e25c7ea104a3ad036e5abcfab8644
Author: Steve <steve@designerwallcoverings.com>
Date:   Wed Jul 22 18:55:10 2026 -0700

    Pipeline: FORCE_REEXTRACT env to re-extract already-snapshotted pages (backfills new fields like images onto unchanged-source homes) + durable grant note
---
 apps/workers/src/pipeline.ts | 40 +++++++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 15 deletions(-)

diff --git a/apps/workers/src/pipeline.ts b/apps/workers/src/pipeline.ts
index 38cf6d9..bc3b193 100644
--- a/apps/workers/src/pipeline.ts
+++ b/apps/workers/src/pipeline.ts
@@ -266,24 +266,34 @@ export async function runPipeline(adapter: SourceAdapter) {
     const existing = await prisma.rawSnapshot.findUnique({
       where: { sourceId_contentHash: { sourceId: source.id, contentHash: page.contentHash } },
     });
-    if (existing) continue; // unchanged bytes → nothing new to extract
+    // FORCE_REEXTRACT=1 re-extracts an already-snapshotted page against its
+    // existing snapshot — used to backfill newly-added fields (e.g. images) onto
+    // homes whose source bytes haven't changed. Default: skip unchanged pages.
+    const force = process.env.FORCE_REEXTRACT === "1";
+    if (existing && !force) continue; // unchanged bytes → nothing new to extract
     summary.changed += 1;
 
-    const storagePath = join(dir, `${page.contentHash}${page.contentType.includes("json") ? ".json" : ".html"}`);
-    await writeFile(storagePath, page.body);
-    const snapshot = await prisma.rawSnapshot.create({
-      data: {
-        sourceId: source.id,
-        url: page.url,
-        retrievedAt: new Date(page.retrievedAt),
-        contentHash: page.contentHash,
-        storagePath,
-        contentType: page.contentType,
-        httpStatus: 200,
-      },
-    });
+    let snapshotId: string;
+    if (existing) {
+      snapshotId = existing.id;
+    } else {
+      const storagePath = join(dir, `${page.contentHash}${page.contentType.includes("json") ? ".json" : ".html"}`);
+      await writeFile(storagePath, page.body);
+      const snapshot = await prisma.rawSnapshot.create({
+        data: {
+          sourceId: source.id,
+          url: page.url,
+          retrievedAt: new Date(page.retrievedAt),
+          contentHash: page.contentHash,
+          storagePath,
+          contentType: page.contentType,
+          httpStatus: 200,
+        },
+      });
+      snapshotId = snapshot.id;
+    }
 
-    const { stagedIds, errors } = await extractStage(adapter, source, page, snapshot.id);
+    const { stagedIds, errors } = await extractStage(adapter, source, page, snapshotId);
     summary.errors.push(...errors);
     summary.staged += stagedIds.length;
 

← 13997e3 Builder images: capture per-home photos from Lennar feed (el  ·  back to Homesonspec  ·  deploy-full.sh: also ship collectors + publisher + schemas s 4be9de7 →