← back to Restaurant Directory
scripts/enrich-cuisine.ts
199 lines
#!/usr/bin/env tsx
/**
* Cuisine classification of every facility via local Ollama qwen3:14b.
*
* Design:
* - Pulls facilities lacking a facility_enrichment row OR with enriched_at IS NULL.
* - Calls Ollama once per facility, parses a small JSON response.
* - Upserts into facility_enrichment.
* - Idempotent: re-runs safely; skip already-enriched (any source).
* - Progress log every 100 records.
* - Emails Steve via George at end (or earlier if >10% failure rate).
*
* Run: npx tsx scripts/enrich-cuisine.ts [--limit N] [--dry-run]
*
* NOTE — local Ollama only, per the standing rule. No Anthropic/OpenAI here.
*/
import { Pool } from 'pg';
const OLLAMA_URL = process.env.OLLAMA_URL ?? 'http://localhost:11434';
const MODEL = process.env.OLLAMA_MODEL ?? 'qwen3:14b';
const GEORGE_URL = process.env.GEORGE_URL ?? 'http://localhost:9850';
const GEORGE_AUTH = process.env.GEORGE_BASIC_AUTH ?? 'admin:DWSecure2024!';
const STEVE_EMAIL = 'steveabramsdesigns@gmail.com';
const args = new Set(process.argv.slice(2));
const dryRun = args.has('--dry-run');
const limitArg = process.argv.find(a => a.startsWith('--limit='));
const LIMIT = limitArg ? parseInt(limitArg.split('=')[1], 10) : Infinity;
const pool = new Pool({
host: process.env.PGHOST ?? '/tmp',
database: process.env.PGDATABASE ?? 'restaurant_professional_directory',
user: process.env.PGUSER ?? process.env.USER,
max: 5,
});
const PROMPT_PREAMBLE = `You classify Los Angeles food-establishment names by cuisine. Return ONLY a single-line JSON object — no markdown, no preamble, no explanation. Schema:
{"cuisine": "<one word or short phrase>", "confidence": <0..1>}
Rules:
- "cuisine" is the cuisine TYPE, not the chain name. "STARBUCKS COFFEE" → "Coffee"; "EL POLLO LOCO" → "Mexican"; "PINK'S HOT DOGS" → "Hot Dogs".
- For unclear or generic names ("JOE'S DELI", "ABC RESTAURANT") return {"cuisine": "American", "confidence": 0.3} — low confidence signals uncertainty.
- For non-restaurants (markets, bakeries, gas stations, schools, hospitals) return {"cuisine": "Non-restaurant", "confidence": 0.9}.
- Common cuisines: American, Mexican, Chinese, Japanese, Korean, Thai, Vietnamese, Indian, Italian, Pizza, Mediterranean, Middle Eastern, Greek, Salvadoran, Peruvian, Cuban, Caribbean, BBQ, Burgers, Hot Dogs, Sandwiches, Coffee, Bakery, Bar, Fast Food, Sushi, Seafood, Steakhouse, Vegan, Breakfast, Doughnuts, Ice Cream, Juice & Smoothie.
- One word is fine. Two words MAX.
- Output ONE LINE OF JSON. Nothing else.`;
type EnrichResult = { cuisine: string; confidence: number } | null;
async function classifyOne(name: string, facility_type: string | null): Promise<EnrichResult> {
const prompt = `${PROMPT_PREAMBLE}\n\nFacility name: "${name}"\nFacility type tag: ${facility_type ?? 'unknown'}\n\nJSON:`;
const res = await fetch(`${OLLAMA_URL}/api/generate`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
model: MODEL,
prompt,
stream: false,
think: false, // qwen3: skip the <think>…</think> reasoning block
options: { temperature: 0.1, num_predict: 80 },
}),
});
if (!res.ok) throw new Error(`ollama ${res.status}`);
const data = await res.json() as { response: string };
const text = (data.response || '').trim();
// qwen3:14b sometimes wraps in <think>…</think>; grab the first {…} block
const match = text.match(/\{[^{}]*"cuisine"[^{}]*\}/);
if (!match) return null;
try {
const parsed = JSON.parse(match[0]);
if (typeof parsed.cuisine !== 'string') return null;
const conf = typeof parsed.confidence === 'number' ? parsed.confidence : 0.5;
return { cuisine: parsed.cuisine.slice(0, 40).trim(), confidence: Math.max(0, Math.min(1, conf)) };
} catch { return null; }
}
async function emailSteve(subject: string, body: string) {
try {
await fetch(`${GEORGE_URL}/api/send?account=info`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
'Authorization': 'Basic ' + Buffer.from(GEORGE_AUTH).toString('base64'),
},
body: JSON.stringify({ to: STEVE_EMAIL, subject, body }),
});
} catch (e) {
console.error('[george] email failed:', (e as Error).message);
}
}
async function main() {
const startedAt = new Date();
console.log(`[enrich] cuisine classification via ${MODEL} starting at ${startedAt.toISOString()}${dryRun ? ' (dry-run)' : ''}`);
// Pull pending facilities
const { rows: pending } = await pool.query<{ facility_id: string; name: string; facility_type: string | null }>(
`SELECT f.facility_id, f.name, f.facility_type
FROM facility f
LEFT JOIN facility_enrichment e ON e.facility_id = f.facility_id
WHERE e.facility_id IS NULL OR e.enriched_at IS NULL
ORDER BY f.facility_id
LIMIT $1`,
[Number.isFinite(LIMIT) ? LIMIT : 1_000_000]
);
console.log(`[enrich] ${pending.length.toLocaleString()} facilities pending`);
let ok = 0, fail = 0, since = Date.now();
for (let i = 0; i < pending.length; i++) {
const f = pending[i];
let result: EnrichResult = null;
try {
result = await classifyOne(f.name, f.facility_type);
} catch (e) {
console.error(`[enrich] ${f.facility_id} ${f.name}: ${(e as Error).message}`);
}
if (!result) {
fail++;
} else {
ok++;
if (!dryRun) {
try {
await pool.query(
`INSERT INTO facility_enrichment (facility_id, cuisine, cuisine_confidence, enriched_at, source)
VALUES ($1, $2, $3, now(), 'qwen3-14b')
ON CONFLICT (facility_id) DO UPDATE SET
cuisine = EXCLUDED.cuisine,
cuisine_confidence = EXCLUDED.cuisine_confidence,
enriched_at = EXCLUDED.enriched_at,
source = EXCLUDED.source`,
[f.facility_id, result.cuisine, result.confidence]
);
} catch (e) {
console.error(`[enrich] ${f.facility_id} upsert failed:`, (e as Error).message);
fail++; ok--;
}
}
}
if ((i + 1) % 100 === 0) {
const dt = (Date.now() - since) / 1000;
const rate = (100 / dt).toFixed(2);
const remain = pending.length - (i + 1);
const eta = (remain / parseFloat(rate) / 60).toFixed(1);
const failRate = (fail / (i + 1) * 100).toFixed(1);
console.log(`[enrich] ${i + 1}/${pending.length} · ok=${ok} fail=${fail} (${failRate}%) · ${rate}/s · ETA ${eta} min`);
since = Date.now();
// Emergency stop if failure rate > 25% in first 500 records
if (i < 500 && fail / (i + 1) > 0.25) {
console.error(`[enrich] ABORT — failure rate ${failRate}% in first ${i + 1}`);
await emailSteve(
`LA County Eats — cuisine enrichment ABORTED at ${i + 1}/${pending.length}`,
`<p>Cuisine enrichment via <code>${MODEL}</code> aborted because failure rate hit ${failRate}% in the first ${i + 1} records (threshold: 25%).</p><p>Likely cause: model not responding or returning malformed JSON.</p><p>Check <code>~/Projects/restaurant-directory/scripts/enrich-cuisine.ts</code> and verify Ollama is running with qwen3:14b loaded.</p>`
);
process.exit(1);
}
}
}
const finishedAt = new Date();
const totalMin = ((finishedAt.getTime() - startedAt.getTime()) / 1000 / 60).toFixed(1);
const finalFailRate = pending.length > 0 ? (fail / pending.length * 100) : 0;
console.log(`[enrich] DONE — ok=${ok} fail=${fail} (${finalFailRate.toFixed(1)}%) in ${totalMin} min`);
await emailSteve(
`LA County Eats — cuisine enrichment ${finalFailRate > 10 ? 'completed WITH WARNINGS' : 'complete'}`,
`<h2>Cuisine enrichment finished</h2>
<ul>
<li>Model: <code>${MODEL}</code></li>
<li>Started: ${startedAt.toISOString()}</li>
<li>Finished: ${finishedAt.toISOString()}</li>
<li>Total runtime: ${totalMin} minutes</li>
<li>Successful: ${ok.toLocaleString()}</li>
<li>Failed: ${fail.toLocaleString()} (${finalFailRate.toFixed(1)}%)</li>
</ul>
${finalFailRate > 10 ? '<p><strong>Failure rate exceeded 10% — investigate.</strong></p>' : ''}
<p>Top cuisines now visible at <a href="https://lacountyeats.com/">lacountyeats.com</a> after the next pm2 restart deploys the cuisine UI.</p>`
);
await pool.query(
`INSERT INTO ingest_run (dataset, source_url, started_at, finished_at,
rows_pulled, rows_inserted, rows_failed)
VALUES ('cuisine_enrichment', $1, $2, $3, $4, $5, $6)`,
[`ollama://${MODEL}`, startedAt, finishedAt, pending.length, ok, fail]
);
await pool.end();
}
main().catch(async (e) => {
console.error('[enrich] FATAL:', e);
await emailSteve('LA County Eats — cuisine enrichment FATAL', `<pre>${(e as Error).stack}</pre>`);
process.exit(1);
});