← back to Trademarks Copyright
src/lib/brandHunter.ts
498 lines
/**
* Brand hunter — finds websites in business 5+ years with NO federal trademark registration.
*
* Pipeline, per candidate domain:
* 1. Fetch homepage HTML (HEAD + GET with timeout).
* 2. Qwen extracts brand_name, established_year, meta_description from HTML excerpt.
* 3. `whois` CLI for domain created date (fallback if model's established_year missing).
* 4. Justia Trademarks search for live federal registrations of the brand name.
* 5. Qwen proposes 5 adjacent fashion-site domains for the discovery queue.
* 6. Row persisted to brand_candidates.
*/
import { execFile as _execFile } from "node:child_process";
import { promisify } from "node:util";
import { query } from "./db";
import { qwenJSON } from "./qwen";
import { webSearch } from "./websearch";
import {
domainLifecycle, abandonmentStatus, opportunityLabel, extractLatestYear,
} from "./lifecycle";
const execFile = promisify(_execFile);
export type HunterEvent =
| { kind: "start"; domain: string }
| { kind: "fetched"; domain: string; status: number }
| { kind: "extracted"; domain: string; brand_name: string | null; established_year: number | null }
| { kind: "whois"; domain: string; created: string | null; years: number | null; expires: string | null }
| { kind: "activity"; domain: string; last_year: number | null; years_stale: number | null; status: string }
| { kind: "uspto"; domain: string; live_marks: number; registered: boolean }
| { kind: "label"; domain: string; label: string }
| { kind: "saved"; domain: string; status: string; years_in_business: number | null; id: number }
| { kind: "skipped"; domain: string; reason: string }
| { kind: "discovered"; parent: string; domains: string[] }
| { kind: "error"; domain: string; message: string }
| { kind: "done"; totals: { discovered: number; verified: number; errors: number } };
const UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
async function fetchHTML(url: string, timeoutMs = 12000): Promise<{ status: number; html: string }> {
const ctrl = new AbortController();
const t = setTimeout(() => ctrl.abort(), timeoutMs);
try {
const r = await fetch(url, {
headers: { "user-agent": UA, accept: "text/html,application/xhtml+xml" },
signal: ctrl.signal,
redirect: "follow",
});
const full = await r.text();
// Keep head (metadata) + tail (footer). Modern sites bury © in the footer
// which gets truncated when we cap from the start.
const head = full.slice(0, 80000);
const tail = full.length > 80000 ? "\n<!--TAIL-->\n" + full.slice(-40000) : "";
return { status: r.status, html: head + tail };
} finally {
clearTimeout(t);
}
}
function pickExcerpt(html: string): string {
// Prefer <title>, <meta description>, and visible header text.
const title = /<title[^>]*>([^<]*)<\/title>/i.exec(html)?.[1]?.trim() ?? "";
const desc = /<meta[^>]+name=["']description["'][^>]+content=["']([^"']+)["']/i.exec(html)?.[1] ?? "";
const ogTitle = /<meta[^>]+property=["']og:title["'][^>]+content=["']([^"']+)["']/i.exec(html)?.[1] ?? "";
const ogSite = /<meta[^>]+property=["']og:site_name["'][^>]+content=["']([^"']+)["']/i.exec(html)?.[1] ?? "";
const h1 = /<h1[^>]*>([\s\S]*?)<\/h1>/i.exec(html)?.[1]?.replace(/<[^>]+>/g, "").trim() ?? "";
const footer = /<footer[\s\S]{0,3000}<\/footer>/i.exec(html)?.[0]?.replace(/<[^>]+>/g, " ").replace(/\s+/g, " ").slice(0, 800) ?? "";
return [
`TITLE: ${title}`,
`OG_SITE: ${ogSite}`,
`OG_TITLE: ${ogTitle}`,
`META_DESC: ${desc}`,
`H1: ${h1}`,
`FOOTER: ${footer}`,
].join("\n");
}
async function extractBrandWithQwen(domain: string, excerpt: string): Promise<{
brand_name: string | null;
established_year: number | null;
meta_description: string | null;
}> {
const prompt = `You are extracting brand info from a website.
Domain: ${domain}
Page excerpt:
${excerpt}
Return STRICT JSON with exactly these keys:
{
"brand_name": string, // the brand/company name as it presents itself
"established_year": number|null, // 4-digit year the business was founded if stated (look for "Since", "Est.", "©", copyright years — pick the earliest credible founding year), else null
"meta_description": string // 1 sentence on what the brand sells
}`;
try {
return await qwenJSON(prompt);
} catch {
return { brand_name: null, established_year: null, meta_description: null };
}
}
async function proposeMoreDomains(category: string, knownDomains: string[]): Promise<string[]> {
const prompt = `You help discover independent ${category} websites.
Already scanned: ${knownDomains.slice(0, 40).join(", ")}.
Propose 8 DIFFERENT real independent ${category} brand websites likely to have been in business 5+ years, but NOT major brands (avoid Nike, Zara, Gucci, Adidas, etc.). Prefer DTC, niche, or independent labels.
Respond with STRICT JSON in this exact shape:
{ "domains": ["example1.com","example2.com", ...] }
Only bare domains (no https, no www, no paths).`;
try {
const obj = await qwenJSON<{ domains?: string[] }>(prompt, 0.7);
const arr = Array.isArray(obj?.domains) ? obj.domains : [];
return arr
.map((d) => String(d).trim().replace(/^https?:\/\//, "").replace(/^www\./, "").replace(/\/.*$/, "").toLowerCase())
.filter((d) => /^[a-z0-9][a-z0-9-]*\.[a-z]{2,}(\.[a-z]{2,})?$/.test(d));
} catch {
return [];
}
}
async function whoisInfo(domain: string): Promise<{ created: Date | null; expires: Date | null }> {
try {
const { stdout } = await execFile("whois", [domain], { timeout: 12000 });
const pickDate = (patterns: RegExp[]) => {
for (const re of patterns) {
const m = re.exec(stdout);
if (m) {
const d = new Date(m[1]);
if (!isNaN(d.getTime())) return d;
}
}
return null;
};
const created = pickDate([
/Creation Date:\s*([0-9]{4}-[0-9]{2}-[0-9]{2})/i,
/Created On:\s*([0-9]{4}-[0-9]{2}-[0-9]{2})/i,
/created:\s*([0-9]{4}-[0-9]{2}-[0-9]{2})/i,
/Registered:\s*([0-9]{4}-[0-9]{2}-[0-9]{2})/i,
]);
const expires = pickDate([
/Registry Expiry Date:\s*([0-9]{4}-[0-9]{2}-[0-9]{2})/i,
/Registrar Registration Expiration Date:\s*([0-9]{4}-[0-9]{2}-[0-9]{2})/i,
/Expiration Date:\s*([0-9]{4}-[0-9]{2}-[0-9]{2})/i,
/Expiry Date:\s*([0-9]{4}-[0-9]{2}-[0-9]{2})/i,
/expires:\s*([0-9]{4}-[0-9]{2}-[0-9]{2})/i,
/Renewal date:\s*([0-9]{4}-[0-9]{2}-[0-9]{2})/i,
]);
return { created, expires };
} catch {
return { created: null, expires: null };
}
}
/**
* USPTO registration check — 100% free, no paid APIs.
* 1. Multi-engine web search (DDG → DDG Lite → Brave), first engine with results wins.
* 2. Feed result snippets to local Qwen.
* 3. Qwen returns structured JSON with live/dead classification + serial numbers.
*/
async function checkUspto(brandName: string, domain: string): Promise<{
live_marks: number;
has_registration: boolean;
url: string;
evidence: string;
unchecked: boolean;
engine: string | null;
}> {
const q = `"${brandName}" trademark USPTO registration status`;
const searchUrl = `https://duckduckgo.com/?q=${encodeURIComponent(q)}`;
const result = await webSearch(q);
if (!result.snippets.length) {
return {
live_marks: -1,
has_registration: false,
url: searchUrl,
evidence: `all search engines failed: ${result.error ?? "no results"}`,
unchecked: true,
engine: null,
};
}
const snippetText = result.snippets.slice(0, 8)
.map((s, i) => `[${i + 1}] ${s.title}\n ${s.url}\n ${s.body}`)
.join("\n");
const prompt = `You are a trademark-status classifier. Given search-engine result snippets about a brand,
decide whether the brand "${brandName}" (website: ${domain}) has a LIVE federal US trademark registration at the USPTO.
"Live" means: a registration that is not cancelled, abandoned, expired, or dead.
Key phrases indicating LIVE: "Registered", "Section 8-Accepted", "Section 15-Accepted", "Live".
Key phrases indicating DEAD: "Cancelled", "Abandoned", "Expired", "Dead", "Section 8-Cancelled".
A pending *application* (no registration number yet) is NOT a live registration — count it as 0.
Only count registrations whose registrant plausibly matches the brand at ${domain}; ignore unrelated same-name marks.
Snippets:
${snippetText}
Respond with STRICT JSON:
{
"has_live_registration": boolean,
"live_mark_count": number,
"registrant": string|null,
"evidence": string,
"serial_numbers": string[]
}`;
type Ans = {
has_live_registration?: boolean;
live_mark_count?: number;
registrant?: string | null;
evidence?: string;
serial_numbers?: string[];
};
try {
const ans = await qwenJSON<Ans>(prompt, 0.1);
const live = !!ans.has_live_registration;
const count = Number.isFinite(ans.live_mark_count) ? Number(ans.live_mark_count) : (live ? 1 : 0);
const evidence = [
ans.evidence ?? "",
ans.registrant ? `Registrant: ${ans.registrant}` : "",
ans.serial_numbers?.length ? `Serials: ${ans.serial_numbers.join(", ")}` : "",
`[via ${result.engine}]`,
].filter(Boolean).join(" · ").slice(0, 500);
return {
live_marks: count,
has_registration: live,
url: searchUrl,
evidence,
unchecked: false,
engine: result.engine,
};
} catch (e) {
return {
live_marks: -1,
has_registration: false,
url: searchUrl,
evidence: `classifier failed: ${(e as Error).message}`,
unchecked: true,
engine: result.engine,
};
}
}
/**
* Brand↔domain alignment sanity check.
* When a domain is parked / redirects to a registrar / points to Shopify's login, Qwen often extracts
* the hosting service's brand ("Hover", "Shopify") instead of the site's own brand. If the extracted
* brand has no token overlap with the domain, we flag it as a parked/redirect and skip the USPTO check.
*/
function isBrandAlignedWithDomain(brand: string | null, domain: string): boolean {
if (!brand) return false;
const normalize = (s: string) => s.toLowerCase().replace(/[^a-z0-9]/g, "");
const domainCore = normalize(domain.split(".")[0]);
const brandCore = normalize(brand);
if (!domainCore || !brandCore) return false;
// Substring match in either direction — handles "BuckMason" ↔ "buckmason", "Cuyana" ↔ "cuyana".
if (domainCore.includes(brandCore) || brandCore.includes(domainCore)) return true;
// Token overlap — handles "Dead Letter Press" ↔ "deadletterpress".
const brandTokens = brand.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length >= 3);
return brandTokens.some((t) => domainCore.includes(t));
}
export type HunterOptions = {
category?: string;
seedDomains: string[];
maxDiscoveries?: number; // cap total new candidates per run
minYearsInBusiness?: number; // default 5
onEvent?: (e: HunterEvent) => void;
};
export async function runBrandHunter(opts: HunterOptions): Promise<{
discovered: number;
verified: number;
errors: number;
}> {
const category = opts.category ?? "fashion";
const cap = opts.maxDiscoveries ?? 30;
const emit = opts.onEvent ?? (() => {});
// minYearsInBusiness retained for future use; currently everything is checked regardless.
void opts.minYearsInBusiness;
const queue = [...new Set(opts.seedDomains.map((d) => d.toLowerCase()))];
const processed = new Set<string>();
let discovered = 0, verified = 0, errors = 0;
// Skip anything already in the DB.
const { rows: existing } = await query<{ domain: string }>(`SELECT domain FROM brand_candidates`);
const known = new Set(existing.map((r) => r.domain));
while (queue.length && discovered < cap) {
const domain = queue.shift()!;
if (processed.has(domain)) continue;
processed.add(domain);
if (known.has(domain)) { emit({ kind: "skipped", domain, reason: "already in db" }); continue; }
// Pace DDG requests — 3-5s between candidates avoids the anomaly page.
if (processed.size > 1) await new Promise((res) => setTimeout(res, 3000 + Math.random() * 2000));
emit({ kind: "start", domain });
try {
// 1. Fetch homepage.
let html = "";
let status = 0;
try {
const resp = await fetchHTML(`https://${domain}`);
status = resp.status; html = resp.html;
emit({ kind: "fetched", domain, status });
} catch {
try {
const resp = await fetchHTML(`http://${domain}`);
status = resp.status; html = resp.html;
emit({ kind: "fetched", domain, status });
} catch (e) {
throw new Error(`fetch failed: ${(e as Error).message}`);
}
}
// 2. Qwen extract.
const excerpt = pickExcerpt(html);
const ex = await extractBrandWithQwen(domain, excerpt);
emit({ kind: "extracted", domain, brand_name: ex.brand_name, established_year: ex.established_year });
const brandName = ex.brand_name || domain.split(".")[0];
// 3. WHOIS (created + expiry).
const who = await whoisInfo(domain);
const establishedYear = ex.established_year ?? (who.created ? who.created.getFullYear() : null);
const yearsInBiz = establishedYear ? new Date().getFullYear() - establishedYear : null;
emit({
kind: "whois",
domain,
created: who.created ? who.created.toISOString().slice(0, 10) : null,
years: yearsInBiz,
expires: who.expires ? who.expires.toISOString().slice(0, 10) : null,
});
// 3b. Activity clock — scan the full HTML (not just excerpt) for the latest year signal.
const activity = extractLatestYear(html);
const currentYear = new Date().getFullYear();
const yearsSinceActivity = activity.year !== null ? currentYear - activity.year : null;
const abStatus = abandonmentStatus(yearsSinceActivity);
emit({
kind: "activity",
domain,
last_year: activity.year,
years_stale: yearsSinceActivity,
status: abStatus,
});
// 3c. Domain lifecycle label.
const lifecycle = domainLifecycle(status, who.expires);
// NOTE: no early exit on age — every candidate gets the USPTO check regardless of
// years_in_business. Age lives on the row as a sortable/filterable column, not a gate.
// Brands under 5 yrs are arguably MORE exposed (no common-law rights built up yet).
// Sanity gate: parked / registrar-redirect pages confuse the extractor. If the extracted
// brand name has no alignment with the domain, skip USPTO and flag as 'parked'.
const aligned = isBrandAlignedWithDomain(ex.brand_name, domain);
let u: Awaited<ReturnType<typeof checkUspto>> | null = null;
let finalStatus: string;
if (!aligned) {
finalStatus = "parked";
emit({
kind: "uspto",
domain,
live_marks: -1,
registered: false,
});
} else {
u = await checkUspto(brandName, domain);
emit({ kind: "uspto", domain, live_marks: u.live_marks, registered: u.has_registration });
finalStatus = u.unchecked
? "pending"
: u.has_registration
? "registered"
: "verified_unregistered";
}
// 5. Compute the opportunity label.
const label = opportunityLabel({
isRegistered: u ? u.has_registration : null,
yearsInBusiness: yearsInBiz,
lifecycle,
abandonment: abStatus,
isParked: !aligned,
});
emit({ kind: "label", domain, label });
// 6. Persist.
const { rows } = await query<{ id: number }>(
`INSERT INTO brand_candidates (
domain, brand_name, category, established_year, years_in_business,
whois_created_date, meta_description,
uspto_checked_at, uspto_live_marks, has_federal_registration, uspto_search_url,
status, suggested_by, suggested_from_domain,
http_status, domain_expires_on, domain_lifecycle,
last_activity_year, last_activity_signal, years_since_activity,
abandonment_status, opportunity_label
) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20,$21,$22)
ON CONFLICT (domain) DO UPDATE SET
brand_name = EXCLUDED.brand_name,
established_year = EXCLUDED.established_year,
years_in_business = EXCLUDED.years_in_business,
uspto_checked_at = EXCLUDED.uspto_checked_at,
uspto_live_marks = EXCLUDED.uspto_live_marks,
has_federal_registration = EXCLUDED.has_federal_registration,
uspto_search_url = EXCLUDED.uspto_search_url,
status = EXCLUDED.status,
http_status = EXCLUDED.http_status,
domain_expires_on = EXCLUDED.domain_expires_on,
domain_lifecycle = EXCLUDED.domain_lifecycle,
last_activity_year = EXCLUDED.last_activity_year,
last_activity_signal = EXCLUDED.last_activity_signal,
years_since_activity = EXCLUDED.years_since_activity,
abandonment_status = EXCLUDED.abandonment_status,
opportunity_label = EXCLUDED.opportunity_label,
last_checked_at = NOW()
RETURNING id`,
[domain, brandName, category, establishedYear, yearsInBiz,
who.created?.toISOString().slice(0, 10) ?? null, ex.meta_description,
aligned ? new Date() : null,
u ? u.live_marks : null,
u ? u.has_registration : null,
u ? u.url : null,
finalStatus, "seed", null,
status || null,
who.expires?.toISOString().slice(0, 10) ?? null,
lifecycle,
activity.year,
activity.signal,
yearsSinceActivity,
abStatus,
label]
);
const noteText = !aligned
? `Extracted brand "${brandName}" does not match domain — likely parked or registrar redirect. USPTO check skipped.`
: u?.evidence ?? "";
if (noteText) {
await query(`UPDATE brand_candidates SET notes = $1 WHERE domain = $2`, [noteText, domain]);
}
emit({ kind: "saved", domain, status: finalStatus, years_in_business: yearsInBiz, id: rows[0].id });
discovered++;
if (finalStatus === "verified_unregistered") verified++;
// 6. Ask Qwen for more.
if (discovered < cap) {
const more = await proposeMoreDomains(category, [...processed, ...queue]);
emit({ kind: "discovered", parent: domain, domains: more });
for (const m of more) {
if (!processed.has(m) && !known.has(m) && !queue.includes(m)) queue.push(m);
}
}
} catch (e) {
errors++;
const msg = e instanceof Error ? e.message : String(e);
emit({ kind: "error", domain, message: msg });
try {
await query(
`INSERT INTO brand_candidates (domain, category, status, error_message, suggested_by)
VALUES ($1,$2,'error',$3,'seed')
ON CONFLICT (domain) DO UPDATE SET status='error', error_message=EXCLUDED.error_message, last_checked_at=NOW()`,
[domain, category, msg]
);
} catch { /* swallow */ }
}
}
emit({ kind: "done", totals: { discovered, verified, errors } });
return { discovered, verified, errors };
}
export const FASHION_SEEDS = [
"outerknown.com",
"cuyana.com",
"everlane.com",
"alexmill.com",
"entireworld.com",
"sezane.com",
"tradlands.com",
"grana.com",
"reformation.com",
"ninelives.com",
"jennikayne.com",
"frankieshop.com",
"kotn.com",
"buckmason.com",
"taylorstitch.com",
"toddsnyder.com",
"flamingos-life.com",
"aloyoga.com",
"girlfriend.com",
"oakandfort.com",
];