← back to Restaurant Directory
scripts/enrich-website-ddg.py
178 lines
#!/usr/bin/env python3
"""Free DuckDuckGo HTML website enricher for lacountyeats facilities.
For each facility without a website, query DDG with `<name> <city> CA restaurant`,
extract the first non-aggregator domain from results, and update facility_enrichment.website.
No API key, no LLM. Pure HTTP + regex. ~1-2 sec per query.
Usage:
python3 enrich-website-ddg.py --limit 200 [--dry-run]
"""
import argparse, re, sys, time, urllib.parse, urllib.request, gzip, io, os
import psycopg
DB_URL = 'postgresql:///restaurant_professional_directory?host=/tmp'
# Aggregators to skip — we want the actual restaurant domain, not Yelp/TripAdvisor
AGGREGATORS = {
'yelp.com', 'tripadvisor.com', 'opentable.com', 'grubhub.com', 'doordash.com',
'ubereats.com', 'postmates.com', 'seamless.com', 'caviar.com', 'mapquest.com',
'foursquare.com', 'zagat.com', 'eater.com', 'thrillist.com', 'timeout.com',
'allmenus.com', 'menupages.com', 'menupix.com', 'menuism.com', 'restaurantji.com',
'facebook.com', 'fb.com', 'instagram.com', 'twitter.com', 'x.com', 'linkedin.com',
'youtube.com', 'pinterest.com', 'tiktok.com',
'yellowpages.com', 'yelp.co', 'maps.google.com', 'google.com', 'bing.com', 'duckduckgo.com',
'wikipedia.org', 'wikidata.org', 'reddit.com',
'wholefoodsmarket.com', # if generic chain
'lacounty.gov', 'ca.gov', 'cdc.gov',
'apple.com', 'amazon.com', 'ebay.com', 'craigslist.org',
'm.facebook.com', 'instagram.com',
'beyondmenu.com', 'slicelife.com', 'chownow.com', 'toasttab.com',
'order.online', 'getbite.com', 'bbb.org',
'archive.org',
'loc8nearme.com', 'restaurantguru.com', 'menuwithprice.com', 'urbanspoon.com',
'zomato.com', 'happycow.net', 'localdatabase.com', 'brownpapertickets.com',
'foodadvisor.com', 'restaurants-near-me.org', 'business.site',
'hotfrog.com', 'manta.com', 'cylex.us.com', 'localedge.com',
'mapsofworld.com', 'expertise.com', 'thumbtack.com', 'angi.com',
'nextdoor.com', 'mybook.com', 'us.kompass.com', 'dnb.com',
'tabelog.com', 'restaurantji.com', 'restaurantji.us',
}
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'
def fetch(url, timeout=8):
req = urllib.request.Request(url, headers={
'User-Agent': USER_AGENT,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9',
'Accept-Encoding': 'gzip, deflate',
'Accept-Language': 'en-US,en;q=0.9',
})
with urllib.request.urlopen(req, timeout=timeout) as resp:
raw = resp.read()
if resp.headers.get('Content-Encoding') == 'gzip':
raw = gzip.decompress(raw)
return raw.decode('utf-8', errors='ignore')
def domain_of(url):
try:
host = urllib.parse.urlparse(url).hostname or ''
host = host.lower().lstrip('www.')
return host
except Exception:
return ''
def is_aggregator(host):
if not host:
return True
for agg in AGGREGATORS:
if host == agg or host.endswith('.' + agg):
return True
return False
# DDG result link pattern — DDG redirects via /l/?uddg=ENCODED_URL or direct https URL
LINK_RE = re.compile(r'class="result__a"[^>]*href="(?P<url>[^"]+)"', re.IGNORECASE)
LINK_RE2 = re.compile(r'<a[^>]+rel="nofollow"[^>]+href="(?P<url>https?://[^"]+)"', re.IGNORECASE)
import json as _json
# Try Brave Search API first (2k/mo free), fall back to DDG
BRAVE_KEY = None
def _load_brave_key():
global BRAVE_KEY
try:
for line in open(os.path.expanduser('~/Projects/secrets-manager/.env')):
if line.startswith('BRAVE_SEARCH_API_KEY='):
BRAVE_KEY = line.split('=', 1)[1].strip().strip('"').strip("'")
break
except Exception:
pass
_load_brave_key()
def brave_search(query):
if not BRAVE_KEY:
return []
q = urllib.parse.quote_plus(query)
url = f'https://api.search.brave.com/res/v1/web/search?q={q}&count=10&country=US'
req = urllib.request.Request(url, headers={
'X-Subscription-Token': BRAVE_KEY,
'Accept': 'application/json',
'User-Agent': USER_AGENT,
})
try:
with urllib.request.urlopen(req, timeout=10) as resp:
data = _json.loads(resp.read())
return [r.get('url', '') for r in (data.get('web', {}).get('results') or [])]
except urllib.error.HTTPError as e:
if e.code == 429:
time.sleep(5)
return []
except Exception:
return []
def ddg_search(query):
"""Brave first, DDG html-fallback. Returns list of candidate URLs."""
urls = brave_search(query)
if urls:
return urls
return [] # DDG html endpoint returning landing page — disable for now
def pick_website(name, city, state):
"""Return best-guess homepage URL or None. Single Brave query is enough; quota is finite."""
q = f'{name} {city} {state} restaurant'
seen_hosts = set()
for url in ddg_search(q):
host = domain_of(url)
if not host or is_aggregator(host) or host in seen_hosts:
seen_hosts.add(host)
continue
seen_hosts.add(host)
return f'https://{host}/'
return None
def main():
ap = argparse.ArgumentParser()
ap.add_argument('--limit', type=int, default=200)
ap.add_argument('--dry-run', action='store_true')
args = ap.parse_args()
with psycopg.connect(DB_URL) as conn:
with conn.cursor() as cur:
cur.execute("""
SELECT f.facility_id, f.name, COALESCE(f.city, ''), COALESCE(f.state, 'CA')
FROM facility f
LEFT JOIN facility_enrichment fe ON fe.facility_id = f.facility_id
WHERE (fe.website IS NULL OR fe.website = '')
AND f.name IS NOT NULL
AND f.city IS NOT NULL
ORDER BY f.facility_id
LIMIT %s
""", (args.limit,))
facilities = cur.fetchall()
print(f'[ddg-enrich] {len(facilities)} facilities to enrich')
hits, misses = 0, 0
for i, (fid, name, city, state) in enumerate(facilities, 1):
website = pick_website(name, city, state)
if website:
hits += 1
if not args.dry_run:
with conn.cursor() as cur:
cur.execute("""
INSERT INTO facility_enrichment (facility_id, website)
VALUES (%s, %s)
ON CONFLICT (facility_id) DO UPDATE SET website = EXCLUDED.website
""", (fid, website))
conn.commit()
print(f' [{i}/{len(facilities)}] ✓ {name[:40]} → {website}')
else:
misses += 1
print(f' [{i}/{len(facilities)}] ✗ {name[:40]}')
sys.stdout.flush()
time.sleep(0.5)
print(f'[ddg-enrich] done · hits={hits} · misses={misses} · hit-rate={100*hits/max(len(facilities),1):.1f}%')
if __name__ == '__main__':
main()