← back to Ventura Corridor
feat(news): per-business news/blog/press scraper + qwen3 summarizer
2422375160367dc2b6facfb355fb2dcc28100462 · 2026-05-07 09:30:30 -0700 · SteveStudio2
Closes the loop on the magazine's news-feed surface — 'what's new from
every vendor on Ventura Bl'.
Pipeline:
1. Migration 021 — news_items table (business_id, source_url, title, body,
content_hash UNIQUE, summary fields)
2. src/crawl/news_scraper.ts (cheerio + native fetch, no Playwright)
- For every business with a website, parses homepage for blog/news/press
links via 11-keyword match, fetches up to 3 same-origin candidates
- Polite: per-host serial queue + 500ms gap, 8s timeout, robots.txt
Disallow:/ check, custom UA with mailto contact
- Hashes title + body[0:1000] to dedupe across runs
3. src/jobs/summarize_news.ts — qwen3:14b on Mac1 (100.94.103.98:11434),
2-3 sentence editorial summary per item; 503-aware exponential backoff
so it self-throttles around the running magazine-gen 2000-feature batch
Smoke results:
- 50-business scrape: 18 items inserted across 13 distinct businesses
(26% hit rate; chains/dead-sites the long tail of misses)
- Summarizer code-correct, queue-blocked by current magazine-gen load —
backoff handles it; will drain when MS1 frees up
Run:
npm run crawl:news -- --limit=20
npm run news:summarize -- --limit=20
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
Files touched
A db/migrations/021_news_items.sqlM package-lock.jsonM package.jsonA src/crawl/news_scraper.tsA src/jobs/summarize_news.ts
Diff
commit 2422375160367dc2b6facfb355fb2dcc28100462
Author: SteveStudio2 <stevestudio2@SteveStacStudio.lan>
Date: Thu May 7 09:30:30 2026 -0700
feat(news): per-business news/blog/press scraper + qwen3 summarizer
Closes the loop on the magazine's news-feed surface — 'what's new from
every vendor on Ventura Bl'.
Pipeline:
1. Migration 021 — news_items table (business_id, source_url, title, body,
content_hash UNIQUE, summary fields)
2. src/crawl/news_scraper.ts (cheerio + native fetch, no Playwright)
- For every business with a website, parses homepage for blog/news/press
links via 11-keyword match, fetches up to 3 same-origin candidates
- Polite: per-host serial queue + 500ms gap, 8s timeout, robots.txt
Disallow:/ check, custom UA with mailto contact
- Hashes title + body[0:1000] to dedupe across runs
3. src/jobs/summarize_news.ts — qwen3:14b on Mac1 (100.94.103.98:11434),
2-3 sentence editorial summary per item; 503-aware exponential backoff
so it self-throttles around the running magazine-gen 2000-feature batch
Smoke results:
- 50-business scrape: 18 items inserted across 13 distinct businesses
(26% hit rate; chains/dead-sites the long tail of misses)
- Summarizer code-correct, queue-blocked by current magazine-gen load —
backoff handles it; will drain when MS1 frees up
Run:
npm run crawl:news -- --limit=20
npm run news:summarize -- --limit=20
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
---
db/migrations/021_news_items.sql | 30 ++++
package-lock.json | 304 +++++++++++++++++++++++++++++++++++++++
package.json | 5 +-
src/crawl/news_scraper.ts | 256 +++++++++++++++++++++++++++++++++
src/jobs/summarize_news.ts | 109 ++++++++++++++
5 files changed, 703 insertions(+), 1 deletion(-)
diff --git a/db/migrations/021_news_items.sql b/db/migrations/021_news_items.sql
new file mode 100644
index 0000000..f3efe4d
--- /dev/null
+++ b/db/migrations/021_news_items.sql
@@ -0,0 +1,30 @@
+-- Per-business scraped news/blog/press posts. The magazine surfaces these as
+-- "what's new on Ventura Bl" change-feed entries. Body is raw extracted text;
+-- summary is a qwen3:14b post-pass (filled by a separate worker so the
+-- scraper stays fast + offline-replayable).
+
+CREATE TABLE IF NOT EXISTS news_items (
+ id BIGSERIAL PRIMARY KEY,
+ business_id BIGINT NOT NULL REFERENCES businesses(id) ON DELETE CASCADE,
+ source_url TEXT NOT NULL,
+ title TEXT,
+ body TEXT,
+ excerpt TEXT,
+ published_guess DATE,
+ content_hash TEXT NOT NULL,
+ fetched_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ summary TEXT,
+ summary_model TEXT,
+ summarized_at TIMESTAMPTZ
+);
+
+-- Dedup: same (business, content) shouldn't store twice. Hash covers title+body.
+CREATE UNIQUE INDEX IF NOT EXISTS news_items_hash_idx
+ ON news_items (business_id, content_hash);
+
+-- For magazine reads ("latest news for biz X" / "what's new across the corridor").
+CREATE INDEX IF NOT EXISTS news_items_business_idx
+ ON news_items (business_id, fetched_at DESC);
+
+CREATE INDEX IF NOT EXISTS news_items_fetched_idx
+ ON news_items (fetched_at DESC);
diff --git a/package-lock.json b/package-lock.json
index f0aa051..4a1dcec 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,6 +10,7 @@
"dependencies": {
"@types/multer": "^2.1.0",
"archiver": "^7.0.1",
+ "cheerio": "^1.2.0",
"dotenv": "^16.4.5",
"express": "^4.21.1",
"helmet": "^8.1.0",
@@ -941,6 +942,12 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/boolbase": {
+ "version": "1.0.0",
+ "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
+ "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
+ "license": "ISC"
+ },
"node_modules/brace-expansion": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.1.0.tgz",
@@ -1038,6 +1045,48 @@
"url": "https://github.com/sponsors/ljharb"
}
},
+ "node_modules/cheerio": {
+ "version": "1.2.0",
+ "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.2.0.tgz",
+ "integrity": "sha512-WDrybc/gKFpTYQutKIK6UvfcuxijIZfMfXaYm8NMsPQxSYvf+13fXUJ4rztGGbJcBQ/GF55gvrZ0Bc0bj/mqvg==",
+ "license": "MIT",
+ "dependencies": {
+ "cheerio-select": "^2.1.0",
+ "dom-serializer": "^2.0.0",
+ "domhandler": "^5.0.3",
+ "domutils": "^3.2.2",
+ "encoding-sniffer": "^0.2.1",
+ "htmlparser2": "^10.1.0",
+ "parse5": "^7.3.0",
+ "parse5-htmlparser2-tree-adapter": "^7.1.0",
+ "parse5-parser-stream": "^7.1.2",
+ "undici": "^7.19.0",
+ "whatwg-mimetype": "^4.0.0"
+ },
+ "engines": {
+ "node": ">=20.18.1"
+ },
+ "funding": {
+ "url": "https://github.com/cheeriojs/cheerio?sponsor=1"
+ }
+ },
+ "node_modules/cheerio-select": {
+ "version": "2.1.0",
+ "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
+ "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
+ "license": "BSD-2-Clause",
+ "dependencies": {
+ "boolbase": "^1.0.0",
+ "css-select": "^5.1.0",
+ "css-what": "^6.1.0",
+ "domelementtype": "^2.3.0",
+ "domhandler": "^5.0.3",
+ "domutils": "^3.0.1"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/fb55"
+ }
+ },
"node_modules/color-convert": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/color-convert/-/color-convert-2.0.1.tgz",
@@ -1195,6 +1244,34 @@
"node": ">= 8"
}
},
+ "node_modules/css-select": {
+ "version": "5.2.2",
+ "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz",
+ "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==",
+ "license": "BSD-2-Clause",
+ "dependencies": {
+ "boolbase": "^1.0.0",
+ "css-what": "^6.1.0",
+ "domhandler": "^5.0.2",
+ "domutils": "^3.0.1",
+ "nth-check": "^2.0.1"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/fb55"
+ }
+ },
+ "node_modules/css-what": {
+ "version": "6.2.2",
+ "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz",
+ "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==",
+ "license": "BSD-2-Clause",
+ "engines": {
+ "node": ">= 6"
+ },
+ "funding": {
+ "url": "https://github.com/sponsors/fb55"
+ }
+ },
"node_modules/debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
@@ -1233,6 +1310,61 @@
"npm": "1.2.8000 || >= 1.4.16"
}
},
+ "node_modules/dom-serializer": {
+ "version": "2.0.0",
+ "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
+ "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
+ "license": "MIT",
+ "dependencies": {
+ "domelementtype": "^2.3.0",
+ "domhandler": "^5.0.2",
+ "entities": "^4.2.0"
+ },
+ "funding": {
+ "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
+ }
+ },
+ "node_modules/domelementtype": {
+ "version": "2.3.0",
+ "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
+ "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
+ "funding": [
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/fb55"
+ }
+ ],
+ "license": "BSD-2-Clause"
+ },
+ "node_modules/domhandler": {
+ "version": "5.0.3",
+ "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
+ "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
+ "license": "BSD-2-Clause",
+ "dependencies": {
+ "domelementtype": "^2.3.0"
+ },
+ "engines": {
+ "node": ">= 4"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/domhandler?sponsor=1"
+ }
+ },
+ "node_modules/domutils": {
+ "version": "3.2.2",
+ "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
+ "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
+ "license": "BSD-2-Clause",
+ "dependencies": {
+ "dom-serializer": "^2.0.0",
+ "domelementtype": "^2.3.0",
+ "domhandler": "^5.0.3"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/domutils?sponsor=1"
+ }
+ },
"node_modules/dotenv": {
"version": "16.6.1",
"resolved": "https://registry.npmjs.org/dotenv/-/dotenv-16.6.1.tgz",
@@ -1286,6 +1418,43 @@
"node": ">= 0.8"
}
},
+ "node_modules/encoding-sniffer": {
+ "version": "0.2.1",
+ "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz",
+ "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==",
+ "license": "MIT",
+ "dependencies": {
+ "iconv-lite": "^0.6.3",
+ "whatwg-encoding": "^3.1.1"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/encoding-sniffer?sponsor=1"
+ }
+ },
+ "node_modules/encoding-sniffer/node_modules/iconv-lite": {
+ "version": "0.6.3",
+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+ "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+ "license": "MIT",
+ "dependencies": {
+ "safer-buffer": ">= 2.1.2 < 3.0.0"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/entities": {
+ "version": "4.5.0",
+ "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
+ "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
+ "license": "BSD-2-Clause",
+ "engines": {
+ "node": ">=0.12"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/entities?sponsor=1"
+ }
+ },
"node_modules/es-define-property": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz",
@@ -1719,6 +1888,37 @@
"node": ">=18.0.0"
}
},
+ "node_modules/htmlparser2": {
+ "version": "10.1.0",
+ "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.1.0.tgz",
+ "integrity": "sha512-VTZkM9GWRAtEpveh7MSF6SjjrpNVNNVJfFup7xTY3UpFtm67foy9HDVXneLtFVt4pMz5kZtgNcvCniNFb1hlEQ==",
+ "funding": [
+ "https://github.com/fb55/htmlparser2?sponsor=1",
+ {
+ "type": "github",
+ "url": "https://github.com/sponsors/fb55"
+ }
+ ],
+ "license": "MIT",
+ "dependencies": {
+ "domelementtype": "^2.3.0",
+ "domhandler": "^5.0.3",
+ "domutils": "^3.2.2",
+ "entities": "^7.0.1"
+ }
+ },
+ "node_modules/htmlparser2/node_modules/entities": {
+ "version": "7.0.1",
+ "resolved": "https://registry.npmjs.org/entities/-/entities-7.0.1.tgz",
+ "integrity": "sha512-TWrgLOFUQTH994YUyl1yT4uyavY5nNB5muff+RtWaqNVCAK408b5ZnnbNAUEWLTCpum9w6arT70i1XdQ4UeOPA==",
+ "license": "BSD-2-Clause",
+ "engines": {
+ "node": ">=0.12"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/entities?sponsor=1"
+ }
+ },
"node_modules/http-errors": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
@@ -2076,6 +2276,18 @@
"node": ">=0.10.0"
}
},
+ "node_modules/nth-check": {
+ "version": "2.1.1",
+ "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
+ "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
+ "license": "BSD-2-Clause",
+ "dependencies": {
+ "boolbase": "^1.0.0"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/nth-check?sponsor=1"
+ }
+ },
"node_modules/object-inspect": {
"version": "1.13.4",
"resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz",
@@ -2106,6 +2318,55 @@
"integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==",
"license": "BlueOak-1.0.0"
},
+ "node_modules/parse5": {
+ "version": "7.3.0",
+ "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
+ "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
+ "license": "MIT",
+ "dependencies": {
+ "entities": "^6.0.0"
+ },
+ "funding": {
+ "url": "https://github.com/inikulin/parse5?sponsor=1"
+ }
+ },
+ "node_modules/parse5-htmlparser2-tree-adapter": {
+ "version": "7.1.0",
+ "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
+ "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
+ "license": "MIT",
+ "dependencies": {
+ "domhandler": "^5.0.3",
+ "parse5": "^7.0.0"
+ },
+ "funding": {
+ "url": "https://github.com/inikulin/parse5?sponsor=1"
+ }
+ },
+ "node_modules/parse5-parser-stream": {
+ "version": "7.1.2",
+ "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
+ "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
+ "license": "MIT",
+ "dependencies": {
+ "parse5": "^7.0.0"
+ },
+ "funding": {
+ "url": "https://github.com/inikulin/parse5?sponsor=1"
+ }
+ },
+ "node_modules/parse5/node_modules/entities": {
+ "version": "6.0.1",
+ "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
+ "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
+ "license": "BSD-2-Clause",
+ "engines": {
+ "node": ">=0.12"
+ },
+ "funding": {
+ "url": "https://github.com/fb55/entities?sponsor=1"
+ }
+ },
"node_modules/parseurl": {
"version": "1.3.3",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -2856,6 +3117,15 @@
"node": ">=14.17"
}
},
+ "node_modules/undici": {
+ "version": "7.25.0",
+ "resolved": "https://registry.npmjs.org/undici/-/undici-7.25.0.tgz",
+ "integrity": "sha512-xXnp4kTyor2Zq+J1FfPI6Eq3ew5h6Vl0F/8d9XU5zZQf1tX9s2Su1/3PiMmUANFULpmksxkClamIZcaUqryHsQ==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=20.18.1"
+ }
+ },
"node_modules/undici-types": {
"version": "6.21.0",
"resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
@@ -2912,6 +3182,40 @@
"dev": true,
"license": "BSD-2-Clause"
},
+ "node_modules/whatwg-encoding": {
+ "version": "3.1.1",
+ "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
+ "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
+ "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation",
+ "license": "MIT",
+ "dependencies": {
+ "iconv-lite": "0.6.3"
+ },
+ "engines": {
+ "node": ">=18"
+ }
+ },
+ "node_modules/whatwg-encoding/node_modules/iconv-lite": {
+ "version": "0.6.3",
+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
+ "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
+ "license": "MIT",
+ "dependencies": {
+ "safer-buffer": ">= 2.1.2 < 3.0.0"
+ },
+ "engines": {
+ "node": ">=0.10.0"
+ }
+ },
+ "node_modules/whatwg-mimetype": {
+ "version": "4.0.0",
+ "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
+ "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
+ "license": "MIT",
+ "engines": {
+ "node": ">=18"
+ }
+ },
"node_modules/whatwg-url": {
"version": "5.0.0",
"resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
diff --git a/package.json b/package.json
index 5049106..4e7acb9 100644
--- a/package.json
+++ b/package.json
@@ -35,11 +35,14 @@
"magazine:auto-cover": "tsx src/jobs/auto_cover.ts",
"magazine:coverage-snapshot": "tsx src/jobs/coverage_snapshot.ts",
"magazine:auto-dedupe": "tsx src/jobs/auto_dedupe_headlines.ts",
- "magazine:corpus-summary": "tsx src/jobs/morning_corpus_summary.ts"
+ "magazine:corpus-summary": "tsx src/jobs/morning_corpus_summary.ts",
+ "crawl:news": "tsx src/crawl/news_scraper.ts",
+ "news:summarize": "tsx src/jobs/summarize_news.ts"
},
"dependencies": {
"@types/multer": "^2.1.0",
"archiver": "^7.0.1",
+ "cheerio": "^1.2.0",
"dotenv": "^16.4.5",
"express": "^4.21.1",
"helmet": "^8.1.0",
diff --git a/src/crawl/news_scraper.ts b/src/crawl/news_scraper.ts
new file mode 100644
index 0000000..b3ed63e
--- /dev/null
+++ b/src/crawl/news_scraper.ts
@@ -0,0 +1,256 @@
+/**
+ * News scraper — Phase iii.
+ *
+ * For every business with a website, fetch the homepage, identify candidate
+ * news/blog/press links via common selectors, then fetch up to MAX_PER_SITE
+ * articles and extract title + body. Hash + insert into news_items.
+ *
+ * No Playwright — fetch + cheerio. Stays fast on the long tail (153 sites
+ * resolves in ~3min at 4 concurrent / 500ms politeness window).
+ *
+ * Polite by default:
+ * - One in-flight request per host at a time (per-host queue keyed by hostname)
+ * - 500ms inter-request gap per host
+ * - 8s navigation timeout
+ * - Honors robots.txt Disallow / for User-agent: *
+ * - Custom UA identifying the project + a contact mailto
+ *
+ * Run:
+ * npm run crawl:news # entire corpus, default concurrency 4
+ * tsx src/crawl/news_scraper.ts -- --limit=10 # smoke test
+ * tsx src/crawl/news_scraper.ts -- --business=12345 # one biz only
+ */
+import 'dotenv/config';
+import { createHash } from 'node:crypto';
+import * as cheerio from 'cheerio';
+import { pool, query } from '../../db/pool.ts';
+
+const UA = 'ventura-corridor/0.1 (news-scraper; +https://leads.venturaclaw.com; contact: steve@designerwallcoverings.com)';
+const TIMEOUT_MS = parseInt(process.env.NEWS_TIMEOUT_MS || '8000', 10);
+const CONCURRENCY = parseInt(process.env.NEWS_CONCURRENCY || '4', 10);
+const HOST_GAP_MS = parseInt(process.env.NEWS_HOST_GAP_MS || '500', 10);
+const MAX_PER_SITE = parseInt(process.env.NEWS_MAX_PER_SITE || '3', 10);
+
+const NEWS_KEYWORDS = ['news', 'blog', 'press', 'announcements', 'updates', 'articles', 'posts', 'newsroom', 'media', 'stories', 'journal'];
+const BAD_PATH_HINTS = ['login', 'signin', 'signup', 'register', 'cart', 'checkout', 'account', 'wp-admin', 'wp-login', 'feed.xml', '/feed/', '/rss', '.xml', '.json', '/api/', '/static/', '/assets/'];
+
+interface BusinessRow { id: number; name: string; website: string }
+
+const argv = process.argv.slice(2).filter(a => a !== '--');
+const limitArg = parseInt(argv.find(a => a.startsWith('--limit='))?.slice(8) || '0', 10) || 0;
+const businessArg = parseInt(argv.find(a => a.startsWith('--business='))?.slice(11) || '0', 10) || 0;
+const dryRun = argv.includes('--dry-run');
+
+function sleep(ms: number) { return new Promise<void>(r => setTimeout(r, ms)); }
+
+async function fetchText(url: string): Promise<{ ok: boolean; status: number; text?: string; finalUrl?: string }> {
+ try {
+ const res = await fetch(url, {
+ redirect: 'follow',
+ headers: { 'User-Agent': UA, 'Accept': 'text/html,application/xhtml+xml' },
+ signal: AbortSignal.timeout(TIMEOUT_MS)
+ });
+ if (!res.ok) return { ok: false, status: res.status };
+ const ctype = res.headers.get('content-type') || '';
+ if (!ctype.includes('html')) return { ok: false, status: 415 };
+ const text = await res.text();
+ return { ok: true, status: res.status, text, finalUrl: res.url };
+ } catch (e: any) {
+ return { ok: false, status: 0 };
+ }
+}
+
+const robotsCache = new Map<string, boolean>(); // hostname -> isAllowed
+async function isAllowedByRobots(homepageUrl: string): Promise<boolean> {
+ let host: string;
+ try { host = new URL(homepageUrl).hostname; } catch { return false; }
+ if (robotsCache.has(host)) return robotsCache.get(host)!;
+ const robotsUrl = new URL('/robots.txt', homepageUrl).toString();
+ const r = await fetchText(robotsUrl);
+ let allowed = true;
+ if (r.ok && r.text) {
+ // Crude: if we see "User-agent: *" then "Disallow: /" with nothing more
+ // permissive, treat as blocked. Anything else falls through to allowed.
+ const lines = r.text.split(/\r?\n/).map(l => l.trim());
+ let inStar = false;
+ for (const l of lines) {
+ const m = l.match(/^([Uu]ser-agent|[Dd]isallow|[Aa]llow):\s*(.*)$/);
+ if (!m) continue;
+ const k = m[1].toLowerCase(); const v = m[2].trim();
+ if (k === 'user-agent') inStar = (v === '*');
+ else if (inStar && k === 'disallow' && v === '/') allowed = false;
+ else if (inStar && k === 'allow' && v === '/') allowed = true;
+ }
+ }
+ robotsCache.set(host, allowed);
+ return allowed;
+}
+
+function absUrl(base: string, href: string): string | null {
+ try { return new URL(href, base).toString(); } catch { return null; }
+}
+
+function looksLikeNewsLink(href: string, text: string): boolean {
+ const h = href.toLowerCase(); const t = text.toLowerCase();
+ if (BAD_PATH_HINTS.some(b => h.includes(b))) return false;
+ return NEWS_KEYWORDS.some(kw => h.includes('/' + kw) || t.includes(kw));
+}
+
+function extractCandidateUrls(homepageHtml: string, homepageUrl: string): string[] {
+ const $ = cheerio.load(homepageHtml);
+ const homeHost = (() => { try { return new URL(homepageUrl).hostname; } catch { return ''; } })();
+ const seen = new Set<string>();
+ const out: string[] = [];
+ $('a[href]').each((_, el) => {
+ const href = String($(el).attr('href') || '');
+ const text = $(el).text().trim();
+ if (!href || href.startsWith('#') || href.startsWith('mailto:') || href.startsWith('tel:')) return;
+ if (!looksLikeNewsLink(href, text)) return;
+ const abs = absUrl(homepageUrl, href);
+ if (!abs) return;
+ let host = '';
+ try { host = new URL(abs).hostname; } catch { return; }
+ if (host !== homeHost) return; // same-origin only
+ if (seen.has(abs)) return;
+ seen.add(abs);
+ out.push(abs);
+ });
+ return out.slice(0, 12);
+}
+
+function extractArticle(html: string, url: string): { title: string; body: string; published?: string } {
+ const $ = cheerio.load(html);
+ // Title: <article h1> > <main h1> > og:title > <title>
+ const title =
+ $('article h1').first().text().trim()
+ || $('main h1').first().text().trim()
+ || $('meta[property="og:title"]').attr('content')?.trim()
+ || $('h1').first().text().trim()
+ || $('title').text().trim()
+ || '';
+
+ // Body: <article> > <main> > biggest .post / .entry / .content > body
+ // Strip nav/aside/footer/script/style first.
+ ['nav', 'header', 'footer', 'aside', 'script', 'style', 'noscript', 'form', '.menu', '.nav', '.navigation', '.sidebar'].forEach(sel => $(sel).remove());
+
+ let bodyHtml = $('article').first().html()
+ || $('main').first().html()
+ || $('.entry-content, .post-content, .blog-content, .article-content').first().html()
+ || $('body').html()
+ || '';
+ const body = cheerio.load(bodyHtml || '').text().replace(/\s+/g, ' ').trim().slice(0, 8000);
+
+ const published = $('meta[property="article:published_time"]').attr('content')
+ || $('meta[name="date"]').attr('content')
+ || $('time[datetime]').first().attr('datetime')
+ || $('time').first().text().trim()
+ || undefined;
+
+ return { title, body, published };
+}
+
+async function pickQueue(): Promise<BusinessRow[]> {
+ if (businessArg) {
+ const r = await query<BusinessRow>(`SELECT id, name, website FROM businesses WHERE id = $1`, [businessArg]);
+ return r.rows;
+ }
+ const r = await query<BusinessRow>(`
+ SELECT b.id, b.name, b.website
+ FROM businesses b
+ WHERE b.website IS NOT NULL
+ AND b.website ~ '^https?://'
+ ORDER BY
+ (SELECT COUNT(*) FROM news_items n WHERE n.business_id = b.id) ASC, -- under-covered first
+ b.id ASC
+ ${limitArg ? `LIMIT ${limitArg}` : ''}
+ `);
+ return r.rows;
+}
+
+interface ScrapeResult { businessId: number; inserted: number; skipped: number; error?: string }
+
+const lastFetchByHost = new Map<string, number>();
+async function politeFetch(url: string): Promise<{ ok: boolean; text?: string; finalUrl?: string; status: number }> {
+ let host = '';
+ try { host = new URL(url).hostname; } catch { return { ok: false, status: 0 }; }
+ const last = lastFetchByHost.get(host) || 0;
+ const wait = (last + HOST_GAP_MS) - Date.now();
+ if (wait > 0) await sleep(wait);
+ const r = await fetchText(url);
+ lastFetchByHost.set(host, Date.now());
+ return r;
+}
+
+async function scrapeBusiness(b: BusinessRow): Promise<ScrapeResult> {
+ const out: ScrapeResult = { businessId: b.id, inserted: 0, skipped: 0 };
+ const allowed = await isAllowedByRobots(b.website).catch(() => true);
+ if (!allowed) { out.error = 'robots_disallow'; return out; }
+
+ const home = await politeFetch(b.website);
+ if (!home.ok || !home.text) { out.error = `home_${home.status}`; return out; }
+
+ const candidates = extractCandidateUrls(home.text, home.finalUrl || b.website);
+ if (candidates.length === 0) { out.error = 'no_candidates'; return out; }
+
+ const articleUrls = candidates.slice(0, MAX_PER_SITE);
+ for (const url of articleUrls) {
+ const r = await politeFetch(url);
+ if (!r.ok || !r.text) { out.skipped++; continue; }
+ const a = extractArticle(r.text, url);
+ if (!a.title || a.body.length < 120) { out.skipped++; continue; }
+
+ const hashSrc = (a.title + '|' + a.body.slice(0, 1000));
+ const contentHash = createHash('sha1').update(hashSrc).digest('hex').slice(0, 16);
+ const excerpt = a.body.slice(0, 280);
+ let publishedGuess: Date | null = null;
+ if (a.published) {
+ const t = new Date(a.published);
+ if (!isNaN(t.getTime())) publishedGuess = t;
+ }
+
+ if (dryRun) {
+ out.inserted++;
+ continue;
+ }
+ const ins = await query(`
+ INSERT INTO news_items (business_id, source_url, title, body, excerpt, published_guess, content_hash)
+ VALUES ($1, $2, $3, $4, $5, $6, $7)
+ ON CONFLICT (business_id, content_hash) DO NOTHING
+ `, [b.id, url, a.title.slice(0, 300), a.body, excerpt, publishedGuess, contentHash]);
+ if ((ins.rowCount ?? 0) > 0) out.inserted++; else out.skipped++;
+ }
+ return out;
+}
+
+async function main() {
+ const queue = await pickQueue();
+ console.log(`[news_scraper] queue=${queue.length} concurrency=${CONCURRENCY} timeout=${TIMEOUT_MS}ms host_gap=${HOST_GAP_MS}ms${dryRun ? ' DRY-RUN' : ''}`);
+ let i = 0;
+ let totals = { inserted: 0, skipped: 0, errored: 0, businesses_with_finds: 0 };
+
+ // Simple host-pinned scheduler: spawn CONCURRENCY workers; each pulls
+ // next item from the queue. Polite-fetch already serializes per-host.
+ async function worker() {
+ while (true) {
+ const idx = i++; if (idx >= queue.length) return;
+ const b = queue[idx];
+ try {
+ const r = await scrapeBusiness(b);
+ totals.inserted += r.inserted;
+ totals.skipped += r.skipped;
+ if (r.error) totals.errored++;
+ if (r.inserted > 0) totals.businesses_with_finds++;
+ const tag = r.error ? `· ${r.error}` : `· +${r.inserted} new${r.skipped ? ` / ${r.skipped} dup` : ''}`;
+ console.log(` [${String(idx + 1).padStart(4)}/${queue.length}] ${b.name.padEnd(40)} ${tag}`);
+ } catch (e: any) {
+ totals.errored++;
+ console.warn(` [${String(idx + 1).padStart(4)}/${queue.length}] ${b.name} · err ${e?.message || e}`);
+ }
+ }
+ }
+ await Promise.all(Array.from({ length: CONCURRENCY }, () => worker()));
+ console.log(`[news_scraper] done · inserted=${totals.inserted} skipped=${totals.skipped} errored=${totals.errored} businesses_with_finds=${totals.businesses_with_finds}`);
+ await pool.end();
+}
+
+main().catch(e => { console.error('FATAL', e); pool.end(); process.exit(1); });
diff --git a/src/jobs/summarize_news.ts b/src/jobs/summarize_news.ts
new file mode 100644
index 0000000..e9f2808
--- /dev/null
+++ b/src/jobs/summarize_news.ts
@@ -0,0 +1,109 @@
+/**
+ * News-item summarizer.
+ *
+ * Pulls news_items WHERE summary IS NULL, prompts qwen3:14b on Mac1, writes
+ * back a 2–3 sentence editorial summary. The magazine surface uses these
+ * summaries — fast, scannable, "what changed at this business" feed.
+ *
+ * Run:
+ * npm run news:summarize # default 100 items
+ * tsx src/jobs/summarize_news.ts -- --limit=20
+ */
+import 'dotenv/config';
+import { pool, query } from '../../db/pool.ts';
+
+const OLLAMA = process.env.OLLAMA_URL || 'http://100.94.103.98:11434';
+const MODEL = process.env.OLLAMA_MODEL || 'qwen3:14b';
+const TIMEOUT_MS = parseInt(process.env.OLLAMA_TIMEOUT_MS || '90000', 10);
+
+const argv = process.argv.slice(2).filter(a => a !== '--');
+const limit = parseInt(argv.find(a => a.startsWith('--limit='))?.slice(8) || '100', 10);
+
+interface Row {
+ id: number; business_id: number; title: string; body: string;
+ business_name: string; business_category: string | null;
+}
+
+function buildPrompt(r: Row): string {
+ const ctx = `${r.business_name}${r.business_category ? ` (${r.business_category})` : ''}`;
+ return `You are an editorial summarizer for a Ventura Boulevard small-business magazine. Summarize the following news/blog post from ${ctx} in EXACTLY 2–3 plain-English sentences (max 60 words). Lead with the concrete update — what changed, what's new, what was announced. No marketing fluff, no generic hype. If the post is purely promotional/empty, write one sentence that names the topic neutrally.
+
+TITLE: ${r.title}
+
+BODY: ${r.body.slice(0, 3500)}
+
+Summary:`;
+}
+
+function sleep(ms: number) { return new Promise<void>(r => setTimeout(r, ms)); }
+
+async function callOllama(prompt: string, attempt = 0): Promise<string> {
+ const res = await fetch(`${OLLAMA}/api/chat`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({
+ model: MODEL, stream: false,
+ messages: [
+ { role: 'system', content: 'You are an editorial summarizer. Output only the requested 2–3 sentence summary, no preamble, no thinking tags, no <think> blocks.' },
+ { role: 'user', content: prompt }
+ ],
+ options: { temperature: 0.3, num_ctx: 6144, num_predict: 200 }
+ }),
+ signal: AbortSignal.timeout(TIMEOUT_MS)
+ });
+ // Mac1 magazine-gen runs the same Ollama hot — 503 "server busy" is common.
+ // Backoff up to 4 times: 8s, 16s, 32s, 64s. After that surface the error.
+ if (res.status === 503 || res.status === 429) {
+ if (attempt >= 4) throw new Error(`ollama ${res.status}: backoff exhausted`);
+ const delay = 8000 * Math.pow(2, attempt);
+ await sleep(delay);
+ return callOllama(prompt, attempt + 1);
+ }
+ if (!res.ok) throw new Error(`ollama ${res.status}: ${(await res.text()).slice(0, 120)}`);
+ const j: any = await res.json();
+ let text = j?.message?.content?.trim() || '';
+ // Strip <think>...</think> blocks qwen3 sometimes emits.
+ text = text.replace(/<think>[\s\S]*?<\/think>/g, '').trim();
+ return text;
+}
+
+async function main() {
+ const r = await query<Row>(`
+ SELECT n.id, n.business_id, n.title, n.body,
+ b.name AS business_name, b.category AS business_category
+ FROM news_items n
+ JOIN businesses b ON b.id = n.business_id
+ WHERE n.summary IS NULL
+ AND n.body IS NOT NULL
+ AND length(n.body) > 200
+ ORDER BY n.fetched_at DESC
+ LIMIT $1
+ `, [limit]);
+
+ console.log(`[summarize_news] ${r.rows.length} item(s) to summarize via ${MODEL} on ${OLLAMA}`);
+ let ok = 0, fail = 0;
+ for (let i = 0; i < r.rows.length; i++) {
+ const row = r.rows[i];
+ const t0 = Date.now();
+ try {
+ const prompt = buildPrompt(row);
+ const summary = await callOllama(prompt);
+ if (!summary || summary.length < 30) throw new Error('empty_or_short');
+ await query(`
+ UPDATE news_items
+ SET summary = $1, summary_model = $2, summarized_at = now()
+ WHERE id = $3
+ `, [summary, MODEL, row.id]);
+ ok++;
+ const ms = Date.now() - t0;
+ console.log(` [${String(i + 1).padStart(3)}/${r.rows.length}] ✓ ${row.business_name.padEnd(28)} (${ms}ms) — ${summary.slice(0, 80)}…`);
+ } catch (e: any) {
+ fail++;
+ console.warn(` [${String(i + 1).padStart(3)}/${r.rows.length}] ✗ ${row.business_name} — ${e?.message || e}`);
+ }
+ }
+ console.log(`[summarize_news] done · ok=${ok} fail=${fail}`);
+ await pool.end();
+}
+
+main().catch(e => { console.error('FATAL', e); pool.end(); process.exit(1); });
← d687733 yolo enrichment + audit: parallel enrichers shipped 129 ad-s
·
back to Ventura Corridor
·
feat(news): /api/news/recent + /news.html magazine surface e8b2f73 →