← back to Ventura Corridor

db/migrations/006_crawl_blocked.sql

18 lines

-- 006_crawl_blocked.sql · flag businesses where the cached HTML is a CDN challenge,
-- not the real site. Their ad_signals + headline are unreliable.

ALTER TABLE business_enrichment
  ADD COLUMN IF NOT EXISTS crawl_blocked TEXT,        -- 'cloudflare' | 'human_verify' | 'rate_limit' | null
  ADD COLUMN IF NOT EXISTS crawl_blocked_at TIMESTAMPTZ;

UPDATE business_enrichment
SET crawl_blocked = 'cloudflare',
    crawl_blocked_at = now(),
    headline = NULL,
    headline_source = NULL,
    headline_voice = NULL,
    headline_voice_label = NULL
WHERE headline ILIKE '%checking your browser%'
   OR headline ILIKE '%verifying you are human%'
   OR headline ILIKE '%just a moment%';